import{a as O,f as X,t as Ae,c as M_,d as nt}from"../chunks/D0-Lpew9.js";import"../chunks/DP5BbXZE.js";import{o as L_}from"../chunks/Cx7tSjKg.js";import{c as tt,g as o_,aI as i_,h as oe,l as Ze,F as a_,i as Q_,L as e,j as Z_,S as N_,k as $t,n as Ne,f as Le,ab as s_,aj as B_,a0 as Ht,a as D_,aJ as xe,b as _t,aK as F_,s as K_,aC as be,aL as $_,ai as wt,aM as H_,aN as Y_,ao as ee,P as Yt,aO as U_,ax as G_,r as l_,p as r_,aP as ht,O as c_,aQ as V_,aR as X_,ag as W_,m as en,d as tn,ad as ft,D as _n,E as nn,aS as on,R as an,ap as m_,t as r,aT as sn,aU as ln,aV as Ie,a8 as d_,aW as rn,aX as Ut,aY as Gt,aZ as cn,a_ as mn,a$ as dn,b0 as pn,b1 as un,b2 as gn,q as hn,aE as bn,u as Vt,b3 as Xt,b4 as fn,b5 as E,aB as yn,v as vn,al as V,b6 as Oe,b7 as wn,x as Y,y as xn,z as h,B as x,A as u,a1 as Ge,b8 as kn,w as Ve}from"../chunks/CdZ30l9o.js";import{a as Pn,d as jn,b as ve,s as B,e as Xe}from"../chunks/CryQBVor.js";import{i as pe}from"../chunks/DHQwSDIs.js";function Se(t,n){return n}function qn(t,n,s){for(var c=[],p=n.length,l,a=n.length,g=0;g<p;g++){let C=n[g];r_(C,()=>{if(l){if(l.pending.delete(C),l.done.add(C),l.pending.size===0){var R=t.outrogroups;yt(t,wt(l.done)),R.delete(l),R.size===0&&(t.outrogroups=null)}}else a-=1},!1)}if(a===0){var y=c.length===0&&s!==null;if(y){var b=s,f=b.parentNode;W_(f),f.append(b),t.items.clear()}yt(t,n,!y)}else l={pending:new Set(n),done:new Set},(t.outrogroups??(t.outrogroups=new Set)).add(l)}function yt(t,n,s=!0){var c;if(t.pending.size>0){c=new Set;for(const a of t.pending.values())for(const g of a)c.add(t.items.get(g).e)}for(var p=0;p<n.length;p++){var l=n[p];if(c!=null&&c.has(l)){l.f|=xe;const a=document.createDocumentFragment();en(l,a)}else tn(n[p],s)}}var Wt;function we(t,n,s,c,p,l=null){var a=t,g=new Map,y=(n&i_)!==0;if(y){var b=t;a=oe?Ze(a_(b)):b.appendChild(tt())}oe&&Q_();var f=null,C=be(()=>{var M=s();return $_(M)?M:M==null?[]:wt(M)}),R,A=new Map,D=!0;function J(M){(U.effect.f&G_)===0&&(U.pending.delete(M),U.fallback=f,Cn(U,R,a,n,c),f!==null&&(R.length===0?(f.f&xe)===0?l_(f):(f.f^=xe,Qe(f,null,a)):r_(f,()=>{f=null})))}function m(M){U.pending.delete(M)}var v=o_(()=>{R=e(C);var M=R.length;let K=!1;if(oe){var ce=Z_(a)===N_;ce!==(M===0)&&(a=$t(),Ze(a),Ne(!1),K=!0)}for(var ie=new Set,W=D_,ue=K_(),ae=0;ae<M;ae+=1){oe&&Le.nodeType===s_&&Le.data===B_&&(a=Le,K=!0,Ne(!1));var se=R[ae],le=c(se,ae),Z=D?null:g.get(le);Z?(Z.v&&Ht(Z.v,se),Z.i&&Ht(Z.i,ae),ue&&W.unskip_effect(Z.e)):(Z=Tn(g,D?a:Wt??(Wt=tt()),se,le,ae,p,n,s),D||(Z.e.f|=xe),g.set(le,Z)),ie.add(le)}if(M===0&&l&&!f&&(D?f=_t(()=>l(a)):(f=_t(()=>l(Wt??(Wt=tt()))),f.f|=xe)),M>ie.size&&F_(),oe&&M>0&&Ze($t()),!D)if(A.set(W,ie),ue){for(const[fe,Re]of g)ie.has(fe)||W.skip_effect(Re.e);W.oncommit(J),W.ondiscard(m)}else J(W);K&&Ne(!0),e(C)}),U={effect:v,items:g,pending:A,outrogroups:null,fallback:f};D=!1,oe&&(a=Le)}function Me(t){for(;t!==null&&(t.f&V_)===0;)t=t.next;return t}function Cn(t,n,s,c,p){var se,le,Z,fe,Re,Je,Ee,Be,De;var l=(c&X_)!==0,a=n.length,g=t.items,y=Me(t.effect.first),b,f=null,C,R=[],A=[],D,J,m,v;if(l)for(v=0;v<a;v+=1)D=n[v],J=p(D,v),m=g.get(J).e,(m.f&xe)===0&&((le=(se=m.nodes)==null?void 0:se.a)==null||le.measure(),(C??(C=new Set)).add(m));for(v=0;v<a;v+=1){if(D=n[v],J=p(D,v),m=g.get(J).e,t.outrogroups!==null)for(const ge of t.outrogroups)ge.pending.delete(m),ge.done.delete(m);if((m.f&ht)!==0&&(l_(m),l&&((fe=(Z=m.nodes)==null?void 0:Z.a)==null||fe.unfix(),(C??(C=new Set)).delete(m))),(m.f&xe)!==0)if(m.f^=xe,m===y)Qe(m,null,s);else{var U=f?f.next:y;m===t.effect.last&&(t.effect.last=m.prev),m.prev&&(m.prev.next=m.next),m.next&&(m.next.prev=m.prev),Te(t,f,m),Te(t,m,U),Qe(m,U,s),f=m,R=[],A=[],y=Me(f.next);continue}if(m!==y){if(b!==void 0&&b.has(m)){if(R.length<A.length){var M=A[0],K;f=M.prev;var ce=R[0],ie=R[R.length-1];for(K=0;K<R.length;K+=1)Qe(R[K],M,s);for(K=0;K<A.length;K+=1)b.delete(A[K]);Te(t,ce.prev,ie.next),Te(t,f,ce),Te(t,ie,M),y=M,f=ie,v-=1,R=[],A=[]}else b.delete(m),Qe(m,y,s),Te(t,m.prev,m.next),Te(t,m,f===null?t.effect.first:f.next),Te(t,f,m),f=m;continue}for(R=[],A=[];y!==null&&y!==m;)(b??(b=new Set)).add(y),A.push(y),y=Me(y.next);if(y===null)continue}(m.f&xe)===0&&R.push(m),f=m,y=Me(m.next)}if(t.outrogroups!==null){for(const ge of t.outrogroups)ge.pending.size===0&&(yt(t,wt(ge.done)),(Re=t.outrogroups)==null||Re.delete(ge));t.outrogroups.size===0&&(t.outrogroups=null)}if(y!==null||b!==void 0){var W=[];if(b!==void 0)for(m of b)(m.f&ht)===0&&W.push(m);for(;y!==null;)(y.f&ht)===0&&y!==t.fallback&&W.push(y),y=Me(y.next);var ue=W.length;if(ue>0){var ae=(c&i_)!==0&&a===0?s:null;if(l){for(v=0;v<ue;v+=1)(Ee=(Je=W[v].nodes)==null?void 0:Je.a)==null||Ee.measure();for(v=0;v<ue;v+=1)(De=(Be=W[v].nodes)==null?void 0:Be.a)==null||De.fix()}qn(t,W,ae)}}l&&c_(()=>{var ge,Fe;if(C!==void 0)for(m of C)(Fe=(ge=m.nodes)==null?void 0:ge.a)==null||Fe.apply()})}function Tn(t,n,s,c,p,l,a,g){var y=(a&H_)!==0?(a&Y_)===0?ee(s,!1,!1):Yt(s):null,b=(a&U_)!==0?Yt(p):null;return{v:y,i:b,e:_t(()=>(l(n,y??s,b??p,g),()=>{t.delete(c)}))}}function Qe(t,n,s){if(t.nodes)for(var c=t.nodes.start,p=t.nodes.end,l=n&&(n.f&xe)===0?n.nodes.start:s;c!==null;){var a=ft(c);if(l.before(c),c===p)return;c=a}}function Te(t,n,s){n===null?t.effect.first=s:n.next=s,s===null?t.effect.last=n:s.prev=n}const En=()=>performance.now(),qe={tick:t=>requestAnimationFrame(t),now:()=>En(),tasks:new Set};function p_(){const t=qe.now();qe.tasks.forEach(n=>{n.c(t)||(qe.tasks.delete(n),n.f())}),qe.tasks.size!==0&&qe.tick(p_)}function Sn(t){let n;return qe.tasks.size===0&&qe.tick(p_),{promise:new Promise(s=>{qe.tasks.add(n={c:t,f:s})}),abort(){qe.tasks.delete(n)}}}function We(t,n){d_(()=>{t.dispatchEvent(new CustomEvent(n))})}function Rn(t){if(t==="float")return"cssFloat";if(t==="offset")return"cssOffset";if(t.startsWith("--"))return t;const n=t.split("-");return n.length===1?n[0]:n[0]+n.slice(1).map(s=>s[0].toUpperCase()+s.slice(1)).join("")}function e_(t){const n={},s=t.split(";");for(const c of s){const[p,l]=c.split(":");if(!p||l===void 0)break;const a=Rn(p.trim());n[a]=l.trim()}return n}const zn=t=>t;function On(t,n,s,c){var m;var p=(t&sn)!==0,l="both",a,g=n.inert,y=n.style.overflow,b,f;function C(){return d_(()=>a??(a=s()(n,(c==null?void 0:c())??{},{direction:l})))}var R={is_global:p,in(){n.inert=g,b=vt(n,C(),f,1,()=>{We(n,"introstart")},()=>{We(n,"introend"),b==null||b.abort(),b=a=void 0,n.style.overflow=y})},out(v){n.inert=!0,f=vt(n,C(),b,0,()=>{We(n,"outrostart")},()=>{We(n,"outroend"),v==null||v()})},stop:()=>{b==null||b.abort(),f==null||f.abort()}},A=_n;if(((m=A.nodes).t??(m.t=[])).push(R),Pn){var D=p;if(!D){for(var J=A.parent;J&&(J.f&nn)!==0;)for(;(J=J.parent)&&(J.f&on)===0;);D=!J||(J.f&an)!==0}D&&m_(()=>{r(()=>R.in())})}}function vt(t,n,s,c,p,l){var a=c===1;if(ln(n)){var g,y=!1;return c_(()=>{if(!y){var v=n({direction:a?"in":"out"});g=vt(t,v,s,c,p,l)}}),{abort:()=>{y=!0,g==null||g.abort()},deactivate:()=>g.deactivate(),reset:()=>g.reset(),t:()=>g.t()}}if(s==null||s.deactivate(),!(n!=null&&n.duration)&&!(n!=null&&n.delay))return p(),l(),{abort:Ie,deactivate:Ie,reset:Ie,t:()=>c};const{delay:b=0,css:f,tick:C,easing:R=zn}=n;var A=[];if(a&&s===void 0&&(C&&C(0,1),f)){var D=e_(f(0,1));A.push(D,D)}var J=()=>1-c,m=t.animate(A,{duration:b,fill:"forwards"});return m.onfinish=()=>{m.cancel(),p();var v=(s==null?void 0:s.t())??1-c;s==null||s.abort();var U=c-v,M=n.duration*Math.abs(U),K=[];if(M>0){var ce=!1;if(f)for(var ie=Math.ceil(M/16.666666666666668),W=0;W<=ie;W+=1){var ue=v+U*R(W/ie),ae=e_(f(ue,1-ue));K.push(ae),ce||(ce=ae.overflow==="hidden")}ce&&(t.style.overflow="hidden"),J=()=>{var se=m.currentTime;return v+U*R(se/M)},C&&Sn(()=>{if(m.playState!=="running")return!1;var se=J();return C(se,1-se),!0})}m=t.animate(K,{duration:M,fill:"forwards"}),m.onfinish=()=>{J=()=>c,C==null||C(c,1-c),l()}},{abort:()=>{m&&(m.cancel(),m.effect=null,m.onfinish=Ie)},deactivate:()=>{l=Ie},reset:()=>{c===0&&(C==null||C(1,0))},t:()=>J()}}function Jn(t,n){let s=null,c=oe;var p;if(oe){s=Le;for(var l=a_(document.head);l!==null&&(l.nodeType!==s_||l.data!==t);)l=ft(l);if(l===null)Ne(!1);else{var a=ft(l);l.remove(),Ze(a)}}oe||(p=document.head.appendChild(tt()));try{o_(()=>{var g=_t(()=>n(p));g.f|=rn})}finally{c&&(Ne(!0),Ze(s))}}const t_=[...` 	
\r\f \v\uFEFF`];function An(t,n,s){var c=t==null?"":""+t;if(s){for(var p of Object.keys(s))if(s[p])c=c?c+" "+p:p;else if(c.length)for(var l=p.length,a=0;(a=c.indexOf(p,a))>=0;){var g=a+l;(a===0||t_.includes(c[a-1]))&&(g===c.length||t_.includes(c[g]))?c=(a===0?"":c.substring(0,a))+c.substring(g+1):a=g}}return c===""?null:c}function In(t,n){return t==null?null:String(t)}function te(t,n,s,c,p,l){var a=t[Ut];if(oe||a!==s||a===void 0){var g=An(s,c,l);(!oe||g!==t.getAttribute("class"))&&(g==null?t.removeAttribute("class"):n?t.className=g:t.setAttribute("class",g)),t[Ut]=s}else if(l&&p!==l)for(var y in l){var b=!!l[y];(p==null||b!==!!p[y])&&t.classList.toggle(y,b)}return l}function et(t,n,s,c){var p=t[Gt];if(oe||p!==n){var l=In(n);(!oe||l!==t.getAttribute("style"))&&(l==null?t.removeAttribute("style"):t.style.cssText=l),t[Gt]=n}return c}const Mn=Symbol("is custom element"),Ln=Symbol("is html"),Qn=gn?"link":"LINK";function S(t,n,s,c){var p=Zn(t);oe&&(p[n]=t.getAttribute(n),n==="src"||n==="srcset"||n==="href"&&t.nodeName===Qn)||p[n]!==(p[n]=s)&&(n==="loading"&&(t[un]=s),s==null?t.removeAttribute(n):typeof s!="string"&&Nn(t).includes(n)?t[n]=s:t.setAttribute(n,s))}function Zn(t){var n;return t[n=cn]??(t[n]={[Mn]:t.nodeName.includes("-"),[Ln]:t.namespaceURI===mn})}var __=new Map;function Nn(t){var n=t.getAttribute("is")||t.nodeName,s=__.get(n);if(s)return s;__.set(n,s=[]);for(var c,p=t,l=Element.prototype;l!==p;){c=pn(p);for(var a in c)c[a].set&&a!=="innerHTML"&&a!=="textContent"&&a!=="innerText"&&s.push(a);p=dn(p)}return s}function Bn(t=!1){const n=hn,s=n.l.u;if(!s)return;let c=()=>E(n.s);if(t){let p=0,l={};const a=yn(()=>{let g=!1;const y=n.s;for(const b in y)y[b]!==l[b]&&(l[b]=y[b],g=!0);return g&&p++,p});c=()=>e(a)}s.b.length&&bn(()=>{n_(n,c),Xt(s.b)}),Vt(()=>{const p=r(()=>s.m.map(fn));return()=>{for(const l of p)typeof l=="function"&&l()}}),s.a.length&&Vt(()=>{n_(n,c),Xt(s.a)})}function n_(t,n){if(t.l.s)for(const s of t.l.s)e(s);n()}function Dn(t){const n=t-1;return n*n*n+1}function Fn(t,{delay:n=0,duration:s=400,easing:c=Dn,axis:p="y"}={}){const l=getComputedStyle(t),a=+l.opacity,g=p==="y"?"height":"width",y=parseFloat(l[g]),b=p==="y"?["top","bottom"]:["left","right"],f=b.map(v=>`${v[0].toUpperCase()}${v.slice(1)}`),C=parseFloat(l[`padding${f[0]}`]),R=parseFloat(l[`padding${f[1]}`]),A=parseFloat(l[`margin${f[0]}`]),D=parseFloat(l[`margin${f[1]}`]),J=parseFloat(l[`border${f[0]}Width`]),m=parseFloat(l[`border${f[1]}Width`]);return{delay:n,duration:s,easing:c,css:v=>`overflow: hidden;opacity: ${Math.min(v*20,1)*a};${g}: ${v*y}px;padding-${b[0]}: ${v*C}px;padding-${b[1]}: ${v*R}px;margin-${b[0]}: ${v*A}px;margin-${b[1]}: ${v*D}px;border-${b[0]}-width: ${v*J}px;border-${b[1]}-width: ${v*m}px;min-${g}: 0`}}const Kn=JSON.parse('{"atom-3-4m__gpt-s-1-4m":{"ci":[1.0725360001654316,2.9948782002109477],"difference":2.0326431301418864,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"baguettotron__atom-3-4m":{"ci":[14.14042731298988,16.348961152329448],"difference":15.263255663614302,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"baguettotron__falcon-h1-tiny-r-90m":{"ci":[7.754579096585064,9.903861793761253],"difference":8.820217466378272,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"baguettotron__gpt-s-1-4m":{"ci":[16.129475466747913,18.474038330502612],"difference":17.295898793756187,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"baguettotron__gpt-s2-5m":{"ci":[9.256397771070153,11.38004880060208],"difference":10.331448058039745,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"baguettotron__gpt2-124m":{"ci":[0.7505292033322868,2.7143772000682045],"difference":1.7402333545016724,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"baguettotron__nanowhale-100m-base":{"ci":[10.704834399063397,12.75788043869504],"difference":11.742736810760391,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"baguettotron__pythia-160m":{"ci":[8.298184537762872,10.43661712641525],"difference":9.394030404387307,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"baguettotron__pythia-31m":{"ci":[11.57570162659012,13.766851356448733],"difference":12.659259145773355,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"baguettotron__slm-10m":{"ci":[8.323210002923028,10.392029505159467],"difference":9.360557666936613,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"baguettotron__supra-50m-base":{"ci":[1.3509633161095662,3.2857258513374914],"difference":2.3278594898349447,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"baguettotron__veyra2-apricot-50m-base":{"ci":[2.5296915340167394,4.5001965180924905],"difference":3.547305330072205,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"falcon-h1-tiny-r-90m__atom-3-4m":{"ci":[5.420215532883351,7.444210689454811],"difference":6.44303819723603,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"falcon-h1-tiny-r-90m__gpt-s-1-4m":{"ci":[7.39930477778452,9.52758940136927],"difference":8.475681327377918,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"falcon-h1-tiny-r-90m__gpt-s2-5m":{"ci":[0.5031066010624538,2.4698466105128163],"difference":1.5112305916614743,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"falcon-h1-tiny-r-90m__nanowhale-100m-base":{"ci":[1.8741543839787467,3.9548193375016245],"difference":2.922519344382119,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"falcon-h1-tiny-r-90m__pythia-160m":{"ci":[-0.580794542209572,1.6693586430525291],"difference":0.5738129380090358,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"tie"},"falcon-h1-tiny-r-90m__pythia-31m":{"ci":[2.730495758942819,4.922286953231492],"difference":3.8390416793950854,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"falcon-h1-tiny-r-90m__slm-10m":{"ci":[-0.4400500092347234,1.5559306125986496],"difference":0.5403402005583414,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"tie"},"gemma-3-270m__atom-3-4m":{"ci":[23.385199210048242,25.661827101907907],"difference":24.542681336414624,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gemma-3-270m__baguettotron":{"ci":[8.231872620662342,10.330917113466759],"difference":9.279425672800324,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gemma-3-270m__falcon-h1-tiny-r-90m":{"ci":[16.984766782761056,19.19024856397875],"difference":18.099643139178596,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gemma-3-270m__gpt-s-1-4m":{"ci":[25.31383386993303,27.774047690234024],"difference":26.57532446655651,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gemma-3-270m__gpt-s2-5m":{"ci":[18.451728069080684,20.73233091916342],"difference":19.61087373084007,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gemma-3-270m__gpt-x2-125m":{"ci":[2.5100180469723314,4.277907176531071],"difference":3.395722404973594,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gemma-3-270m__gpt2-124m":{"ci":[10.033805987347693,12.02164981680341],"difference":11.019659027301998,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gemma-3-270m__gpt2-medium-355m":{"ci":[4.567690555799388,6.414205704409783],"difference":5.490700049071364,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gemma-3-270m__limen-0-2b":{"ci":[1.346747725245972,3.162401660917923],"difference":2.257917152787524,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gemma-3-270m__mobilellm-r1-140m-base":{"ci":[6.7285212269252535,8.654045161821886],"difference":7.708492950574182,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gemma-3-270m__nanowhale-100m-base":{"ci":[19.911079186076847,22.136911037973743],"difference":21.022162483560713,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gemma-3-270m__pythia-160m":{"ci":[17.49071148889552,19.77861550252012],"difference":18.67345607718763,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gemma-3-270m__pythia-31m":{"ci":[20.72644303179628,23.094161479902546],"difference":21.93868481857368,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gemma-3-270m__slm-10m":{"ci":[17.546740227831336,19.736908231495857],"difference":18.639983339736936,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gemma-3-270m__smollm2-135m":{"ci":[0.16836524011662418,1.9651071736191295],"difference":1.0576894080699517,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gemma-3-270m__supra-50m-base":{"ci":[10.628267365916303,12.551452199235303],"difference":11.60728516263527,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gemma-3-270m__veyra2-apricot-50m-base":{"ci":[11.848671246541473,13.81413489824284],"difference":12.82673100287253,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt-s2-5m__atom-3-4m":{"ci":[4.048447096687848,5.816152073030026],"difference":4.931807605574556,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt-s2-5m__gpt-s-1-4m":{"ci":[5.994355598920947,7.87200612832699],"difference":6.964450735716443,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt-s2-5m__nanowhale-100m-base":{"ci":[0.48907344006903475,2.3424471779865126],"difference":1.4112887527206444,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt-s2-5m__pythia-31m":{"ci":[1.3601371630505645,3.270819706973676],"difference":2.3278110877336107,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt-x2-125m__atom-3-4m":{"ci":[20.055298011374177,22.273700065241986],"difference":21.146958931441034,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt-x2-125m__baguettotron":{"ci":[4.919037209936775,6.853051793908139],"difference":5.883703267826731,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt-x2-125m__falcon-h1-tiny-r-90m":{"ci":[13.662735239486574,15.718124447804271],"difference":14.703920734205003,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt-x2-125m__gpt-s-1-4m":{"ci":[22.00694789823436,24.304170912361922],"difference":23.17960206158292,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt-x2-125m__gpt-s2-5m":{"ci":[15.169372508200368,17.236224767654043],"difference":16.215151325866476,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt-x2-125m__gpt2-124m":{"ci":[6.747015411597489,8.522372794247302],"difference":7.623936622328404,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt-x2-125m__gpt2-medium-355m":{"ci":[1.2408356393255757,2.9406022879415636],"difference":2.094977644097771,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt-x2-125m__mobilellm-r1-140m-base":{"ci":[3.4001537890801874,5.2268186213654895],"difference":4.31277054560059,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt-x2-125m__nanowhale-100m-base":{"ci":[16.613503795581103,18.6676187860639],"difference":17.626440078587123,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt-x2-125m__pythia-160m":{"ci":[14.206915778777304,16.37469514048295],"difference":15.277733672214039,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt-x2-125m__pythia-31m":{"ci":[17.423412908889823,19.65878482694102],"difference":18.542962413600087,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt-x2-125m__slm-10m":{"ci":[14.224111495050982,16.29911483056891],"difference":15.244260934763345,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt-x2-125m__supra-50m-base":{"ci":[7.358523605020158,9.038699445436707],"difference":8.211562757661675,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt-x2-125m__veyra2-apricot-50m-base":{"ci":[8.539745385852653,10.31133646972243],"difference":9.431008597898938,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt2-124m__atom-3-4m":{"ci":[12.414672706067238,14.51910829641859],"difference":13.523022309112628,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt2-124m__falcon-h1-tiny-r-90m":{"ci":[6.088666706845199,8.086574541183792],"difference":7.079984111876598,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt2-124m__gpt-s-1-4m":{"ci":[14.470574653613818,16.628850295397264],"difference":15.555665439254517,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt2-124m__gpt-s2-5m":{"ci":[7.590743978294489,9.543732770147578],"difference":8.591214703538071,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt2-124m__nanowhale-100m-base":{"ci":[9.06687024657007,10.940810819228764],"difference":10.002503456258717,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt2-124m__pythia-160m":{"ci":[6.665984868286377,8.642212289460703],"difference":7.653797049885634,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt2-124m__pythia-31m":{"ci":[9.944853555192541,11.916576975073232],"difference":10.919025791271682,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt2-124m__slm-10m":{"ci":[6.635890360218745,8.602630427540772],"difference":7.620324312434939,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt2-124m__supra-50m-base":{"ci":[-0.22231016068905624,1.4204925503246024],"difference":0.5876261353332725,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"tie"},"gpt2-124m__veyra2-apricot-50m-base":{"ci":[0.9211716297796128,2.705342689004217],"difference":1.8070719755705327,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt2-medium-355m__atom-3-4m":{"ci":[17.993914360231898,20.11495959677222],"difference":19.05198128734326,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt2-medium-355m__baguettotron":{"ci":[2.7853069505419072,4.8012688019468905],"difference":3.788725623728961,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt2-medium-355m__falcon-h1-tiny-r-90m":{"ci":[11.59984492132836,13.659119500473565],"difference":12.608943090107232,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt2-medium-355m__gpt-s-1-4m":{"ci":[19.901446108663194,22.191881707503878],"difference":21.084624417485145,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt2-medium-355m__gpt-s2-5m":{"ci":[13.110875827961552,15.153969564470772],"difference":14.120173681768705,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt2-medium-355m__gpt2-124m":{"ci":[4.700135778835568,6.333309904394746],"difference":5.528958978230634,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt2-medium-355m__mobilellm-r1-140m-base":{"ci":[1.2852382146811867,3.19745496440504],"difference":2.217792901502818,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt2-medium-355m__nanowhale-100m-base":{"ci":[14.524544922835736,16.532400611219693],"difference":15.531462434489349,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt2-medium-355m__pythia-160m":{"ci":[12.147837559283435,14.24955981177643],"difference":13.182756028116266,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt2-medium-355m__pythia-31m":{"ci":[15.388049395302364,17.515758670665342],"difference":16.44798476950232,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt2-medium-355m__slm-10m":{"ci":[12.13887142432881,14.182253685153766],"difference":13.149283290665572,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt2-medium-355m__supra-50m-base":{"ci":[5.190257140204221,7.02033532538955],"difference":6.1165851135639056,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt2-medium-355m__veyra2-apricot-50m-base":{"ci":[6.391542401843375,8.29118001567598],"difference":7.336030953801166,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"lfm2-350m__atom-3-4m":{"ci":[27.58893756755036,30.042187485115143],"difference":28.789996315485123,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"lfm2-350m__baguettotron":{"ci":[12.443170128806557,14.63607099576427],"difference":13.526740651870819,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"lfm2-350m__falcon-h1-tiny-r-90m":{"ci":[21.180647286515022,23.545436928921546],"difference":22.34695811824909,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"lfm2-350m__gemma-3-270m":{"ci":[3.2295084969038235,5.224620472855046],"difference":4.247314979070493,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"lfm2-350m__gpt-s-1-4m":{"ci":[29.546430185030548,32.048984499243076],"difference":30.822639445627008,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"lfm2-350m__gpt-s2-5m":{"ci":[22.692921220894892,25.060730091228283],"difference":23.858188709910568,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"lfm2-350m__gpt-x2-125m":{"ci":[6.543548887485459,8.719161211825703],"difference":7.643037384044087,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"lfm2-350m__gpt2-124m":{"ci":[14.143222563888733,16.4118593800478],"difference":15.266974006372491,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"lfm2-350m__gpt2-medium-355m":{"ci":[8.647105187970773,10.81188758495598],"difference":9.738015028141858,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"lfm2-350m__limen-0-2b":{"ci":[5.465900728782017,7.5273454316989],"difference":6.505232131858017,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"lfm2-350m__mobilellm-r1-140m-base":{"ci":[10.873284350857205,13.02640742452202],"difference":11.955807929644676,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"lfm2-350m__nanowhale-100m-base":{"ci":[24.094314926003207,26.46428926562647],"difference":25.269477462631208,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"lfm2-350m__pythia-160m":{"ci":[21.76448313300386,24.12912979103644],"difference":22.920771056258122,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"lfm2-350m__pythia-31m":{"ci":[24.947883218281085,27.411184938190967],"difference":26.185999797644175,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"lfm2-350m__qwen2-5-0-5b":{"ci":[-0.7937443045238336,1.2109056554752804],"difference":0.19496537145225148,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"tie"},"lfm2-350m__slm-10m":{"ci":[21.736389241473272,24.094793643932555],"difference":22.887298318807428,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"lfm2-350m__smollm2-135m":{"ci":[4.2639180400644365,6.304742227756106],"difference":5.305004387140445,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"lfm2-350m__supra-50m-base":{"ci":[14.717188310254356,16.936606168548764],"difference":15.854600141705763,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"lfm2-350m__veyra2-apricot-50m-base":{"ci":[15.922087105664474,18.21523002545234],"difference":17.074045981943023,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"limen-0-2b__atom-3-4m":{"ci":[21.161513441941704,23.390632501383145],"difference":22.2847641836271,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"limen-0-2b__baguettotron":{"ci":[6.060289304833157,8.008273883388364],"difference":7.021508520012802,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"limen-0-2b__falcon-h1-tiny-r-90m":{"ci":[14.749412938163642,16.841796663719702],"difference":15.841725986391072,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"limen-0-2b__gpt-s-1-4m":{"ci":[23.10948466843261,25.458602842321564],"difference":24.317407313768992,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"limen-0-2b__gpt-s2-5m":{"ci":[16.30800521073441,18.4279354737554],"difference":17.352956578052545,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"limen-0-2b__gpt-x2-125m":{"ci":[0.3224073766022845,1.9122064121523048],"difference":1.1378052521860695,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"limen-0-2b__gpt2-124m":{"ci":[7.833099108366243,9.65152584643135],"difference":8.761741874514474,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"limen-0-2b__gpt2-medium-355m":{"ci":[2.3561414181964686,4.07347368481736],"difference":3.2327828962838403,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"limen-0-2b__mobilellm-r1-140m-base":{"ci":[4.488179280316571,6.39340909504688],"difference":5.450575797786659,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"limen-0-2b__nanowhale-100m-base":{"ci":[17.773543836990466,19.818671756113275],"difference":18.764245330773193,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"limen-0-2b__pythia-160m":{"ci":[15.347024605257046,17.461363940919785],"difference":16.415538924400106,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"limen-0-2b__pythia-31m":{"ci":[18.600823907513195,20.77696639320947],"difference":19.68076766578616,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"limen-0-2b__slm-10m":{"ci":[15.304067721870874,17.457018849422926],"difference":16.382066186949416,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"limen-0-2b__supra-50m-base":{"ci":[8.485989855643487,10.252743940672737],"difference":9.349368009847748,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"limen-0-2b__veyra2-apricot-50m-base":{"ci":[9.680541335952592,11.49796980911933],"difference":10.568813850085007,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"mobilellm-r1-140m-base__atom-3-4m":{"ci":[15.724389468592625,17.9181576661227],"difference":16.83418838584044,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"mobilellm-r1-140m-base__baguettotron":{"ci":[0.6028456289645157,2.5682754519275246],"difference":1.5709327222261427,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"mobilellm-r1-140m-base__falcon-h1-tiny-r-90m":{"ci":[9.366761121353298,11.456567965416465],"difference":10.391150188604414,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"mobilellm-r1-140m-base__gpt-s-1-4m":{"ci":[17.72035213429454,19.952356120653356],"difference":18.86683151598233,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"mobilellm-r1-140m-base__gpt-s2-5m":{"ci":[10.837860366434668,12.916752286983545],"difference":11.902380780265888,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"mobilellm-r1-140m-base__gpt2-124m":{"ci":[2.3527040000432247,4.2604573197727165],"difference":3.3111660767278153,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"mobilellm-r1-140m-base__nanowhale-100m-base":{"ci":[12.244547029232189,14.383753052584806],"difference":13.313669532986532,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"mobilellm-r1-140m-base__pythia-160m":{"ci":[9.850460315653072,12.110903533239728],"difference":10.96496312661345,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"mobilellm-r1-140m-base__pythia-31m":{"ci":[13.173488573335227,15.260343800248487],"difference":14.230191867999498,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"mobilellm-r1-140m-base__slm-10m":{"ci":[9.834008908382621,11.962969758352465],"difference":10.931490389162754,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"mobilellm-r1-140m-base__supra-50m-base":{"ci":[2.9471642992203297,4.800180307345809],"difference":3.898792212061088,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"mobilellm-r1-140m-base__veyra2-apricot-50m-base":{"ci":[4.204117362859041,6.037751350789785],"difference":5.118238052298348,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"nanowhale-100m-base__atom-3-4m":{"ci":[2.5114800270993958,4.522962101886533],"difference":3.520518852853912,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"nanowhale-100m-base__gpt-s-1-4m":{"ci":[4.520390817955864,6.571451905864101],"difference":5.553161982995798,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"nanowhale-100m-base__pythia-31m":{"ci":[-0.04982526813426013,1.8445162857873265],"difference":0.9165223350129663,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"tie"},"pythia-160m__atom-3-4m":{"ci":[4.737172915984907,7.018566565651529],"difference":5.8692252592269964,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"pythia-160m__gpt-s-1-4m":{"ci":[6.721804247693422,9.058153968300607],"difference":7.901868389368882,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"pythia-160m__gpt-s2-5m":{"ci":[-0.19073668035679597,2.031592137727357],"difference":0.9374176536524387,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"tie"},"pythia-160m__nanowhale-100m-base":{"ci":[1.24315052655882,3.4148330794518693],"difference":2.348706406373083,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"pythia-160m__pythia-31m":{"ci":[2.148599543800454,4.357216741831833],"difference":3.2652287413860495,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"pythia-31m__atom-3-4m":{"ci":[1.6057485508016338,3.598754829970397],"difference":2.6039965178409457,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"pythia-31m__gpt-s-1-4m":{"ci":[3.553404497040384,5.681548302264185],"difference":4.636639647982832,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"qwen2-5-0-5b__atom-3-4m":{"ci":[27.429065503800224,29.758126578860814],"difference":28.59503094403287,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"qwen2-5-0-5b__baguettotron":{"ci":[12.233474837251677,14.410463289394922],"difference":13.331775280418565,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"qwen2-5-0-5b__falcon-h1-tiny-r-90m":{"ci":[21.042544193912452,23.292755794541097],"difference":22.151992746796836,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"qwen2-5-0-5b__gemma-3-270m":{"ci":[3.1289293385506296,4.949670349757808],"difference":4.0523496076182415,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"qwen2-5-0-5b__gpt-s-1-4m":{"ci":[29.358591326483403,31.787910953580926],"difference":30.627674074174756,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"qwen2-5-0-5b__gpt-s2-5m":{"ci":[22.484367687263422,24.80643089248962],"difference":23.663223338458312,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"qwen2-5-0-5b__gpt-x2-125m":{"ci":[6.504515401914428,8.401385534393647],"difference":7.448072012591836,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"qwen2-5-0-5b__gpt2-124m":{"ci":[14.119663769992147,16.08811450435093],"difference":15.07200863492024,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"qwen2-5-0-5b__gpt2-medium-355m":{"ci":[8.580291489301949,10.482138750922191],"difference":9.543049656689606,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"qwen2-5-0-5b__limen-0-2b":{"ci":[5.395933348530456,7.243893177924733],"difference":6.310266760405766,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"qwen2-5-0-5b__mobilellm-r1-140m-base":{"ci":[10.797843611952915,12.719193410831675],"difference":11.760842558192424,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"qwen2-5-0-5b__nanowhale-100m-base":{"ci":[23.99988085171323,26.190262920767886],"difference":25.074512091178956,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"qwen2-5-0-5b__pythia-160m":{"ci":[21.56004563074552,23.87905437950645],"difference":22.725805684805874,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"qwen2-5-0-5b__pythia-31m":{"ci":[24.841628881519235,27.14392774955085],"difference":25.991034426191924,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"qwen2-5-0-5b__slm-10m":{"ci":[21.573153354252433,23.791472841951883],"difference":22.69233294735518,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"qwen2-5-0-5b__smollm2-135m":{"ci":[4.182377294645956,6.041752403489344],"difference":5.110039015688194,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"qwen2-5-0-5b__supra-50m-base":{"ci":[14.674719371285,16.64642100282309],"difference":15.659634770253513,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"qwen2-5-0-5b__veyra2-apricot-50m-base":{"ci":[15.843063351497843,17.88305005638959],"difference":16.87908061049077,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"slm-10m__atom-3-4m":{"ci":[5.036334220813094,6.855128264261082],"difference":5.902697996677689,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"slm-10m__gpt-s-1-4m":{"ci":[6.926706396733146,8.94055735587147],"difference":7.935341126819576,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"slm-10m__gpt-s2-5m":{"ci":[0.06659882140217428,1.8222234812887246],"difference":0.970890391103133,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"slm-10m__nanowhale-100m-base":{"ci":[1.4914905145647108,3.2592393440485696],"difference":2.382179143823778,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"slm-10m__pythia-160m":{"ci":[-1.0366806397870105,1.1714324189900007],"difference":0.03347273745069435,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"tie"},"slm-10m__pythia-31m":{"ci":[2.372327463312847,4.223426615282993],"difference":3.2987014788367444,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"smollm2-135m__atom-3-4m":{"ci":[22.34756564971431,24.615212198542363],"difference":23.484991928344677,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"smollm2-135m__baguettotron":{"ci":[7.21981046049257,9.250291115821636],"difference":8.221736264730373,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"smollm2-135m__falcon-h1-tiny-r-90m":{"ci":[15.998703266315285,18.097428607290865],"difference":17.04195373110864,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"smollm2-135m__gpt-s-1-4m":{"ci":[24.306597878265215,26.660924664410892],"difference":25.517635058486558,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"smollm2-135m__gpt-s2-5m":{"ci":[17.474818949264733,19.65927462835377],"difference":18.553184322770118,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"smollm2-135m__gpt-x2-125m":{"ci":[1.520836731078651,3.1770188989552963],"difference":2.338032996903642,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"smollm2-135m__gpt2-124m":{"ci":[9.01471391018264,10.912959371692486],"difference":9.961969619232047,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"smollm2-135m__gpt2-medium-355m":{"ci":[3.5516102335971445,5.330605430074946],"difference":4.433010641001412,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"smollm2-135m__limen-0-2b":{"ci":[0.3179865353874399,2.0083468477308863],"difference":1.200227744717572,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"smollm2-135m__mobilellm-r1-140m-base":{"ci":[5.754466156999673,7.575045374739086],"difference":6.65080354250423,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"smollm2-135m__nanowhale-100m-base":{"ci":[18.89929834073857,21.076546736195468],"difference":19.964473075490766,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"smollm2-135m__pythia-160m":{"ci":[16.533556862312427,18.762458693268172],"difference":17.61576666911768,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"smollm2-135m__pythia-31m":{"ci":[19.743832355893147,21.98849774424981],"difference":20.88099541050373,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"smollm2-135m__slm-10m":{"ci":[16.48898449494742,18.664225888028522],"difference":17.58229393166699,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"smollm2-135m__supra-50m-base":{"ci":[9.632456329100759,11.448445329599902],"difference":10.549595754565319,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"smollm2-135m__veyra2-apricot-50m-base":{"ci":[10.828312999027693,12.721441869861339],"difference":11.769041594802578,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"supra-50m-base__atom-3-4m":{"ci":[11.943975471387882,13.975141551589562],"difference":12.935396173779356,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"supra-50m-base__falcon-h1-tiny-r-90m":{"ci":[5.4933752538658185,7.494091690632844],"difference":6.4923579765433255,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"supra-50m-base__gpt-s-1-4m":{"ci":[13.913111223226107,16.026314781815554],"difference":14.968039303921243,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"supra-50m-base__gpt-s2-5m":{"ci":[7.055711440473382,8.973542914177806],"difference":8.0035885682048,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"supra-50m-base__nanowhale-100m-base":{"ci":[8.481265325554629,10.332046076912933],"difference":9.414877320925445,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"supra-50m-base__pythia-160m":{"ci":[5.988387846530205,8.160833376393306],"difference":7.0661709145523615,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"supra-50m-base__pythia-31m":{"ci":[9.341259784482242,11.317334294187644],"difference":10.33139965593841,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"supra-50m-base__slm-10m":{"ci":[6.057935337460307,8.006326905907825],"difference":7.032698177101667,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"supra-50m-base__veyra2-apricot-50m-base":{"ci":[0.352626714684538,2.1370426725718357],"difference":1.21944584023726,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"veyra2-apricot-50m-base__atom-3-4m":{"ci":[10.627668222625106,12.751269936412879],"difference":11.715950333542096,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"veyra2-apricot-50m-base__falcon-h1-tiny-r-90m":{"ci":[4.269879876321943,6.237640792156269],"difference":5.272912136306066,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"veyra2-apricot-50m-base__gpt-s-1-4m":{"ci":[12.499782605339313,14.838887566245907],"difference":13.748593463683981,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"veyra2-apricot-50m-base__gpt-s2-5m":{"ci":[5.8032698087522885,7.724554386452487],"difference":6.784142727967539,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"veyra2-apricot-50m-base__nanowhale-100m-base":{"ci":[7.226025832871106,9.190466152028234],"difference":8.195431480688185,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"veyra2-apricot-50m-base__pythia-160m":{"ci":[4.834200955167154,6.909557402281555],"difference":5.846725074315101,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"veyra2-apricot-50m-base__pythia-31m":{"ci":[8.142263967111687,10.097664917317006],"difference":9.11195381570115,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"veyra2-apricot-50m-base__slm-10m":{"ci":[4.863643976660482,6.735987982048942],"difference":5.813252336864407,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"}}'),$n="2026-07-27T17:06:56.669003+00:00",Hn=JSON.parse('[{"artifacts":{"result_file":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/lfm2-350m/20260716T154647Z-v1.1/results_v1.1_merged.json","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/lfm2-350m/20260716T154647Z-v1.1"},"benchmarks":{"arc_challenge":{"accuracy":0.39590443686006827,"chance":0.25,"ci":[15.696814562002272,23.321956769055745],"classification":"reliably above chance","label":"ARC-Challenge","n_samples":1172,"score":19.45392491467577},"arc_easy":{"accuracy":0.6641414141414141,"chance":0.25,"ci":[52.637485970819306,57.744107744107744],"classification":"reliably above chance","label":"ARC-Easy","n_samples":2376,"score":55.21885521885522},"blimp":{"accuracy":0.7906593253968254,"categories":{"Anaphor agreement":0.977,"Argument structure":0.7756666666666666,"Binding":0.7418571428571428,"Control / raising":0.8074,"Determiner-noun agreement":0.9377500000000001,"Ellipsis":0.8654999999999999,"Filler-gap":0.7779999999999999,"Irregular forms":0.728,"Island effects":0.739,"NPI licensing":0.5485714285714286,"Quantifiers":0.7414999999999999,"Subject-verb agreement":0.8476666666666666},"category_count":12,"chance":0.5,"ci":[57.22199222883596,58.42621676587302],"classification":"reliably above chance","label":"BLiMP","n_samples":70000,"score":58.13186507936508},"commonsense_qa":{"accuracy":0.45045045045045046,"chance":0.2,"ci":[27.927927927927925,34.88943488943489],"classification":"reliably above chance","label":"CommonsenseQA","n_samples":1221,"score":31.306306306306304},"hellaswag":{"accuracy":0.4898426608245369,"chance":0.25,"ci":[30.61143198566022,33.25433180641306],"classification":"reliably above chance","label":"HellaSwag","n_samples":10042,"score":31.97902144327159},"piqa":{"accuracy":0.6958650707290533,"chance":0.5,"ci":[35.03808487486397,43.2018498367791],"classification":"reliably above chance","label":"PIQA","n_samples":1838,"score":39.17301414581067}},"complete":true,"composite":{"ci":[42.03623437890757,44.21094708361269],"classification":"reliably above chance","score":43.17876224644195},"id":"lfm2-350m","name":"LFM2 350M","params_m":354.484,"protocol":{"bootstrap_replicates":2000,"bootstrap_seed":20260716,"harness":"lm-eval==0.4.12","harness_git_hash":null,"inference_config":{"batch_size":"auto","batch_sizes":[64],"bootstrap_iters":100000,"device":"cuda:0","fewshot_seed":1234,"gen_kwargs":{},"limit":null,"model":"hf","model_args":{"dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false},"model_dtype":"torch.bfloat16","model_num_parameters":354483968,"model_revision":"main","model_sha":"","numpy_seed":1234,"random_seed":1234,"tasks":["hellaswag","piqa","arc_easy","arc_challenge","commonsense_qa_text","blimp"],"torch_seed":1234,"use_cache":null,"v1_1_sources":{"rechecked_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/lfm2-350m/20260716T154341Z-v1.1/results_v1.1_merged.json","unchanged_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/lfm2-350m/20260716T154341Z-v1.1/results_v1.1_merged.json"}},"n_shot":{"arc_easy":0,"blimp":0,"blimp_adjunct_island":0,"blimp_anaphor_gender_agreement":0,"blimp_anaphor_number_agreement":0,"blimp_animate_subject_passive":0,"blimp_animate_subject_trans":0,"blimp_causative":0,"blimp_complex_NP_island":0,"blimp_coordinate_structure_constraint_complex_left_branch":0,"blimp_coordinate_structure_constraint_object_extraction":0,"blimp_determiner_noun_agreement_1":0,"blimp_determiner_noun_agreement_2":0,"blimp_determiner_noun_agreement_irregular_1":0,"blimp_determiner_noun_agreement_irregular_2":0,"blimp_determiner_noun_agreement_with_adj_2":0,"blimp_determiner_noun_agreement_with_adj_irregular_1":0,"blimp_determiner_noun_agreement_with_adj_irregular_2":0,"blimp_determiner_noun_agreement_with_adjective_1":0,"blimp_distractor_agreement_relational_noun":0,"blimp_distractor_agreement_relative_clause":0,"blimp_drop_argument":0,"blimp_ellipsis_n_bar_1":0,"blimp_ellipsis_n_bar_2":0,"blimp_existential_there_object_raising":0,"blimp_existential_there_quantifiers_1":0,"blimp_existential_there_quantifiers_2":0,"blimp_existential_there_subject_raising":0,"blimp_expletive_it_object_raising":0,"blimp_inchoative":0,"blimp_intransitive":0,"blimp_irregular_past_participle_adjectives":0,"blimp_irregular_past_participle_verbs":0,"blimp_irregular_plural_subject_verb_agreement_1":0,"blimp_irregular_plural_subject_verb_agreement_2":0,"blimp_left_branch_island_echo_question":0,"blimp_left_branch_island_simple_question":0,"blimp_matrix_question_npi_licensor_present":0,"blimp_npi_present_1":0,"blimp_npi_present_2":0,"blimp_only_npi_licensor_present":0,"blimp_only_npi_scope":0,"blimp_passive_1":0,"blimp_passive_2":0,"blimp_principle_A_c_command":0,"blimp_principle_A_case_1":0,"blimp_principle_A_case_2":0,"blimp_principle_A_domain_1":0,"blimp_principle_A_domain_2":0,"blimp_principle_A_domain_3":0,"blimp_principle_A_reconstruction":0,"blimp_regular_plural_subject_verb_agreement_1":0,"blimp_regular_plural_subject_verb_agreement_2":0,"blimp_sentential_negation_npi_licensor_present":0,"blimp_sentential_negation_npi_scope":0,"blimp_sentential_subject_island":0,"blimp_superlative_quantifiers_1":0,"blimp_superlative_quantifiers_2":0,"blimp_tough_vs_raising_1":0,"blimp_tough_vs_raising_2":0,"blimp_transitive":0,"blimp_wh_island":0,"blimp_wh_questions_object_gap":0,"blimp_wh_questions_subject_gap":0,"blimp_wh_questions_subject_gap_long_distance":0,"blimp_wh_vs_that_no_gap":0,"blimp_wh_vs_that_no_gap_long_distance":0,"blimp_wh_vs_that_with_gap":0,"blimp_wh_vs_that_with_gap_long_distance":0,"commonsense_qa":0,"hellaswag":0,"piqa":0},"suite":"slm-benchmark-v1.1","task_config":{"arc_challenge":{"dataset_name":"ARC-Challenge","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_challenge.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_challenge","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"arc_easy":{"dataset_name":"ARC-Easy","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_easy.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_easy","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"blimp_adjunct_island":{"dataset_name":"adjunct_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/adjunct_island.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_adjunct_island","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_gender_agreement":{"dataset_name":"anaphor_gender_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_gender_agreement","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_number_agreement":{"dataset_name":"anaphor_number_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_number_agreement.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_number_agreement","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_passive":{"dataset_name":"animate_subject_passive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_passive.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_passive","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_trans":{"dataset_name":"animate_subject_trans","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_trans.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_trans","unsafe_code":false,"validation_split":"train"},"blimp_causative":{"dataset_name":"causative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/causative.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_causative","unsafe_code":false,"validation_split":"train"},"blimp_complex_NP_island":{"dataset_name":"complex_NP_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/complex_NP_island.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_complex_NP_island","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_complex_left_branch":{"dataset_name":"coordinate_structure_constraint_complex_left_branch","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_complex_left_branch","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_object_extraction":{"dataset_name":"coordinate_structure_constraint_object_extraction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_object_extraction","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_1":{"dataset_name":"determiner_noun_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_2":{"dataset_name":"determiner_noun_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_1":{"dataset_name":"determiner_noun_agreement_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_2":{"dataset_name":"determiner_noun_agreement_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_2":{"dataset_name":"determiner_noun_agreement_with_adj_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_1":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_2":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adjective_1":{"dataset_name":"determiner_noun_agreement_with_adjective_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adjective_1","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relational_noun":{"dataset_name":"distractor_agreement_relational_noun","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relational_noun","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relative_clause":{"dataset_name":"distractor_agreement_relative_clause","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relative_clause","unsafe_code":false,"validation_split":"train"},"blimp_drop_argument":{"dataset_name":"drop_argument","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/drop_argument.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_drop_argument","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_1":{"dataset_name":"ellipsis_n_bar_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_1","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_2":{"dataset_name":"ellipsis_n_bar_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_object_raising":{"dataset_name":"existential_there_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_object_raising.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_1":{"dataset_name":"existential_there_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_2":{"dataset_name":"existential_there_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_subject_raising":{"dataset_name":"existential_there_subject_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_subject_raising.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_subject_raising","unsafe_code":false,"validation_split":"train"},"blimp_expletive_it_object_raising":{"dataset_name":"expletive_it_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/expletive_it_object_raising.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_expletive_it_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_inchoative":{"dataset_name":"inchoative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/inchoative.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_inchoative","unsafe_code":false,"validation_split":"train"},"blimp_intransitive":{"dataset_name":"intransitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/intransitive.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_intransitive","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_adjectives":{"dataset_name":"irregular_past_participle_adjectives","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_adjectives","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_verbs":{"dataset_name":"irregular_past_participle_verbs","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_verbs","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_1":{"dataset_name":"irregular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_2":{"dataset_name":"irregular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_echo_question":{"dataset_name":"left_branch_island_echo_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_echo_question","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_simple_question":{"dataset_name":"left_branch_island_simple_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_simple_question","unsafe_code":false,"validation_split":"train"},"blimp_matrix_question_npi_licensor_present":{"dataset_name":"matrix_question_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_matrix_question_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_1":{"dataset_name":"npi_present_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_1.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_1","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_2":{"dataset_name":"npi_present_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_2.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_2","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_licensor_present":{"dataset_name":"only_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_scope":{"dataset_name":"only_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_passive_1":{"dataset_name":"passive_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_1.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_1","unsafe_code":false,"validation_split":"train"},"blimp_passive_2":{"dataset_name":"passive_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_2.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_c_command":{"dataset_name":"principle_A_c_command","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_c_command.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_c_command","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_1":{"dataset_name":"principle_A_case_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_1.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_2":{"dataset_name":"principle_A_case_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_2.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_1":{"dataset_name":"principle_A_domain_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_1.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_2":{"dataset_name":"principle_A_domain_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_2.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_3":{"dataset_name":"principle_A_domain_3","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_3.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_3","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_reconstruction":{"dataset_name":"principle_A_reconstruction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_reconstruction.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_reconstruction","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_1":{"dataset_name":"regular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_2":{"dataset_name":"regular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_licensor_present":{"dataset_name":"sentential_negation_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_scope":{"dataset_name":"sentential_negation_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_sentential_subject_island":{"dataset_name":"sentential_subject_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_subject_island.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_subject_island","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_1":{"dataset_name":"superlative_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_2":{"dataset_name":"superlative_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_1":{"dataset_name":"tough_vs_raising_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_1.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_1","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_2":{"dataset_name":"tough_vs_raising_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_2.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_2","unsafe_code":false,"validation_split":"train"},"blimp_transitive":{"dataset_name":"transitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/transitive.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_transitive","unsafe_code":false,"validation_split":"train"},"blimp_wh_island":{"dataset_name":"wh_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_island.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_island","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_object_gap":{"dataset_name":"wh_questions_object_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_object_gap.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_object_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap":{"dataset_name":"wh_questions_subject_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap_long_distance":{"dataset_name":"wh_questions_subject_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap":{"dataset_name":"wh_vs_that_no_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap_long_distance":{"dataset_name":"wh_vs_that_no_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap":{"dataset_name":"wh_vs_that_with_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap_long_distance":{"dataset_name":"wh_vs_that_with_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap_long_distance","unsafe_code":false,"validation_split":"train"},"commonsense_qa_text":{"dataset_path":"tau/commonsense_qa","description":"","doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/config/tasks/commonsense_qa_text.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"commonsense_qa_text","training_split":"train","unsafe_code":false,"validation_split":"validation"},"hellaswag":{"dataset_path":"Rowan/hellaswag","description":"","doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_config":{"doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":"<function process_docs at 0x7e746d33d940>","sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","process_docs":"def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\\n    def _process_doc(doc):\\n        ctx = doc[\\"ctx_a\\"] + \\" \\" + doc[\\"ctx_b\\"].capitalize()\\n        out_doc = {\\n            \\"query\\": preprocess(doc[\\"activity_label\\"] + \\": \\" + ctx),\\n            \\"choices\\": [preprocess(ending) for ending in doc[\\"endings\\"]],\\n            \\"gold\\": int(doc[\\"label\\"]),\\n        }\\n        return out_doc\\n\\n    return dataset.map(_process_doc)\\n","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"hellaswag","training_split":"train","unsafe_code":false,"validation_split":"validation"},"piqa":{"dataset_path":"baber/piqa","description":"","doc_to_choice":"{{[sol1, sol2]}}","doc_to_decontamination_query":"goal","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{[sol1, sol2]}}","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/piqa/piqa.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"piqa","training_split":"train","unsafe_code":false,"validation_split":"validation"}},"task_versions":{"arc_challenge":1,"arc_easy":1,"blimp":"2.0","blimp_adjunct_island":1,"blimp_anaphor_gender_agreement":1,"blimp_anaphor_number_agreement":1,"blimp_animate_subject_passive":1,"blimp_animate_subject_trans":1,"blimp_causative":1,"blimp_complex_NP_island":1,"blimp_coordinate_structure_constraint_complex_left_branch":1,"blimp_coordinate_structure_constraint_object_extraction":1,"blimp_determiner_noun_agreement_1":1,"blimp_determiner_noun_agreement_2":1,"blimp_determiner_noun_agreement_irregular_1":1,"blimp_determiner_noun_agreement_irregular_2":1,"blimp_determiner_noun_agreement_with_adj_2":1,"blimp_determiner_noun_agreement_with_adj_irregular_1":1,"blimp_determiner_noun_agreement_with_adj_irregular_2":1,"blimp_determiner_noun_agreement_with_adjective_1":1,"blimp_distractor_agreement_relational_noun":1,"blimp_distractor_agreement_relative_clause":1,"blimp_drop_argument":1,"blimp_ellipsis_n_bar_1":1,"blimp_ellipsis_n_bar_2":1,"blimp_existential_there_object_raising":1,"blimp_existential_there_quantifiers_1":1,"blimp_existential_there_quantifiers_2":1,"blimp_existential_there_subject_raising":1,"blimp_expletive_it_object_raising":1,"blimp_inchoative":1,"blimp_intransitive":1,"blimp_irregular_past_participle_adjectives":1,"blimp_irregular_past_participle_verbs":1,"blimp_irregular_plural_subject_verb_agreement_1":1,"blimp_irregular_plural_subject_verb_agreement_2":1,"blimp_left_branch_island_echo_question":1,"blimp_left_branch_island_simple_question":1,"blimp_matrix_question_npi_licensor_present":1,"blimp_npi_present_1":1,"blimp_npi_present_2":1,"blimp_only_npi_licensor_present":1,"blimp_only_npi_scope":1,"blimp_passive_1":1,"blimp_passive_2":1,"blimp_principle_A_c_command":1,"blimp_principle_A_case_1":1,"blimp_principle_A_case_2":1,"blimp_principle_A_domain_1":1,"blimp_principle_A_domain_2":1,"blimp_principle_A_domain_3":1,"blimp_principle_A_reconstruction":1,"blimp_regular_plural_subject_verb_agreement_1":1,"blimp_regular_plural_subject_verb_agreement_2":1,"blimp_sentential_negation_npi_licensor_present":1,"blimp_sentential_negation_npi_scope":1,"blimp_sentential_subject_island":1,"blimp_superlative_quantifiers_1":1,"blimp_superlative_quantifiers_2":1,"blimp_tough_vs_raising_1":1,"blimp_tough_vs_raising_2":1,"blimp_transitive":1,"blimp_wh_island":1,"blimp_wh_questions_object_gap":1,"blimp_wh_questions_subject_gap":1,"blimp_wh_questions_subject_gap_long_distance":1,"blimp_wh_vs_that_no_gap":1,"blimp_wh_vs_that_no_gap_long_distance":1,"blimp_wh_vs_that_with_gap":1,"blimp_wh_vs_that_with_gap_long_distance":1,"commonsense_qa_text":"Yaml","hellaswag":1,"piqa":1}},"rank_range":[1,2],"release_date":"2025-07-10","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/lfm2-350m/20260716T154647Z-v1.1","source":"LiquidAI/LFM2-350M","track":"instruction","updated_at":"2026-07-16T15:46:57.444261+00:00","vocab_size":65536},{"artifacts":{"result_file":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/qwen2-5-0-5b/20260716T154647Z-v1.1/results_v1.1_merged.json","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/qwen2-5-0-5b/20260716T154647Z-v1.1"},"benchmarks":{"arc_challenge":{"accuracy":0.3199658703071672,"chance":0.25,"ci":[5.802047781569963,13.083048919226394],"classification":"reliably above chance","label":"ARC-Challenge","n_samples":1172,"score":9.328782707622297},"arc_easy":{"accuracy":0.5845959595959596,"chance":0.25,"ci":[41.86307519640854,47.1955667789001],"classification":"reliably above chance","label":"ARC-Easy","n_samples":2376,"score":44.61279461279461},"blimp":{"accuracy":0.8435170304232805,"categories":{"Anaphor agreement":0.986,"Argument structure":0.8395555555555556,"Binding":0.7834285714285715,"Control / raising":0.8469999999999999,"Determiner-noun agreement":0.95575,"Ellipsis":0.8895,"Filler-gap":0.7902857142857144,"Irregular forms":0.9515,"Island effects":0.694125,"NPI licensing":0.7531428571428572,"Quantifiers":0.7372500000000001,"Subject-verb agreement":0.8946666666666667},"category_count":12,"chance":0.5,"ci":[67.90119460978836,68.90242162698412],"classification":"reliably above chance","label":"BLiMP","n_samples":70000,"score":68.70340608465611},"commonsense_qa":{"accuracy":0.4095004095004095,"chance":0.2,"ci":[22.91154791154791,29.67086404586403],"classification":"reliably above chance","label":"CommonsenseQA","n_samples":1221,"score":26.187551187551186},"hellaswag":{"accuracy":0.5212109141605258,"chance":0.25,"ci":[34.846976033990565,37.50282148310429],"classification":"reliably above chance","label":"HellaSwag","n_samples":10042,"score":36.161455221403436},"piqa":{"accuracy":0.6942328618063112,"chance":0.5,"ci":[34.71164309031556,42.763873775843294],"classification":"reliably above chance","label":"PIQA","n_samples":1838,"score":38.84657236126223}},"complete":true,"composite":{"ci":[41.86348233604306,43.96668012300082],"classification":"reliably above chance","score":42.989729225325554},"id":"qwen2-5-0-5b","name":"Qwen2.5 0.5B","params_m":494,"protocol":{"bootstrap_replicates":2000,"bootstrap_seed":20260716,"harness":"lm-eval==0.4.12","harness_git_hash":null,"inference_config":{"batch_size":"auto","batch_sizes":[64],"bootstrap_iters":100000,"device":"cuda:0","fewshot_seed":1234,"gen_kwargs":{},"limit":null,"model":"hf","model_args":{"dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false},"model_dtype":"torch.bfloat16","model_num_parameters":494032768,"model_revision":"main","model_sha":"","numpy_seed":1234,"random_seed":1234,"tasks":["hellaswag","piqa","arc_easy","arc_challenge","commonsense_qa_text","blimp"],"torch_seed":1234,"use_cache":null,"v1_1_sources":{"rechecked_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/qwen2-5-0-5b/20260716T154341Z-v1.1/results_v1.1_merged.json","unchanged_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/qwen2-5-0-5b/20260716T154341Z-v1.1/results_v1.1_merged.json"}},"n_shot":{"arc_easy":0,"blimp":0,"blimp_adjunct_island":0,"blimp_anaphor_gender_agreement":0,"blimp_anaphor_number_agreement":0,"blimp_animate_subject_passive":0,"blimp_animate_subject_trans":0,"blimp_causative":0,"blimp_complex_NP_island":0,"blimp_coordinate_structure_constraint_complex_left_branch":0,"blimp_coordinate_structure_constraint_object_extraction":0,"blimp_determiner_noun_agreement_1":0,"blimp_determiner_noun_agreement_2":0,"blimp_determiner_noun_agreement_irregular_1":0,"blimp_determiner_noun_agreement_irregular_2":0,"blimp_determiner_noun_agreement_with_adj_2":0,"blimp_determiner_noun_agreement_with_adj_irregular_1":0,"blimp_determiner_noun_agreement_with_adj_irregular_2":0,"blimp_determiner_noun_agreement_with_adjective_1":0,"blimp_distractor_agreement_relational_noun":0,"blimp_distractor_agreement_relative_clause":0,"blimp_drop_argument":0,"blimp_ellipsis_n_bar_1":0,"blimp_ellipsis_n_bar_2":0,"blimp_existential_there_object_raising":0,"blimp_existential_there_quantifiers_1":0,"blimp_existential_there_quantifiers_2":0,"blimp_existential_there_subject_raising":0,"blimp_expletive_it_object_raising":0,"blimp_inchoative":0,"blimp_intransitive":0,"blimp_irregular_past_participle_adjectives":0,"blimp_irregular_past_participle_verbs":0,"blimp_irregular_plural_subject_verb_agreement_1":0,"blimp_irregular_plural_subject_verb_agreement_2":0,"blimp_left_branch_island_echo_question":0,"blimp_left_branch_island_simple_question":0,"blimp_matrix_question_npi_licensor_present":0,"blimp_npi_present_1":0,"blimp_npi_present_2":0,"blimp_only_npi_licensor_present":0,"blimp_only_npi_scope":0,"blimp_passive_1":0,"blimp_passive_2":0,"blimp_principle_A_c_command":0,"blimp_principle_A_case_1":0,"blimp_principle_A_case_2":0,"blimp_principle_A_domain_1":0,"blimp_principle_A_domain_2":0,"blimp_principle_A_domain_3":0,"blimp_principle_A_reconstruction":0,"blimp_regular_plural_subject_verb_agreement_1":0,"blimp_regular_plural_subject_verb_agreement_2":0,"blimp_sentential_negation_npi_licensor_present":0,"blimp_sentential_negation_npi_scope":0,"blimp_sentential_subject_island":0,"blimp_superlative_quantifiers_1":0,"blimp_superlative_quantifiers_2":0,"blimp_tough_vs_raising_1":0,"blimp_tough_vs_raising_2":0,"blimp_transitive":0,"blimp_wh_island":0,"blimp_wh_questions_object_gap":0,"blimp_wh_questions_subject_gap":0,"blimp_wh_questions_subject_gap_long_distance":0,"blimp_wh_vs_that_no_gap":0,"blimp_wh_vs_that_no_gap_long_distance":0,"blimp_wh_vs_that_with_gap":0,"blimp_wh_vs_that_with_gap_long_distance":0,"commonsense_qa":0,"hellaswag":0,"piqa":0},"suite":"slm-benchmark-v1.1","task_config":{"arc_challenge":{"dataset_name":"ARC-Challenge","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_challenge.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_challenge","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"arc_easy":{"dataset_name":"ARC-Easy","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_easy.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_easy","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"blimp_adjunct_island":{"dataset_name":"adjunct_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/adjunct_island.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_adjunct_island","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_gender_agreement":{"dataset_name":"anaphor_gender_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_gender_agreement","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_number_agreement":{"dataset_name":"anaphor_number_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_number_agreement.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_number_agreement","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_passive":{"dataset_name":"animate_subject_passive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_passive.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_passive","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_trans":{"dataset_name":"animate_subject_trans","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_trans.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_trans","unsafe_code":false,"validation_split":"train"},"blimp_causative":{"dataset_name":"causative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/causative.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_causative","unsafe_code":false,"validation_split":"train"},"blimp_complex_NP_island":{"dataset_name":"complex_NP_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/complex_NP_island.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_complex_NP_island","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_complex_left_branch":{"dataset_name":"coordinate_structure_constraint_complex_left_branch","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_complex_left_branch","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_object_extraction":{"dataset_name":"coordinate_structure_constraint_object_extraction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_object_extraction","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_1":{"dataset_name":"determiner_noun_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_2":{"dataset_name":"determiner_noun_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_1":{"dataset_name":"determiner_noun_agreement_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_2":{"dataset_name":"determiner_noun_agreement_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_2":{"dataset_name":"determiner_noun_agreement_with_adj_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_1":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_2":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adjective_1":{"dataset_name":"determiner_noun_agreement_with_adjective_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adjective_1","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relational_noun":{"dataset_name":"distractor_agreement_relational_noun","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relational_noun","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relative_clause":{"dataset_name":"distractor_agreement_relative_clause","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relative_clause","unsafe_code":false,"validation_split":"train"},"blimp_drop_argument":{"dataset_name":"drop_argument","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/drop_argument.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_drop_argument","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_1":{"dataset_name":"ellipsis_n_bar_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_1","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_2":{"dataset_name":"ellipsis_n_bar_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_object_raising":{"dataset_name":"existential_there_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_object_raising.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_1":{"dataset_name":"existential_there_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_2":{"dataset_name":"existential_there_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_subject_raising":{"dataset_name":"existential_there_subject_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_subject_raising.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_subject_raising","unsafe_code":false,"validation_split":"train"},"blimp_expletive_it_object_raising":{"dataset_name":"expletive_it_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/expletive_it_object_raising.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_expletive_it_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_inchoative":{"dataset_name":"inchoative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/inchoative.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_inchoative","unsafe_code":false,"validation_split":"train"},"blimp_intransitive":{"dataset_name":"intransitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/intransitive.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_intransitive","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_adjectives":{"dataset_name":"irregular_past_participle_adjectives","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_adjectives","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_verbs":{"dataset_name":"irregular_past_participle_verbs","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_verbs","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_1":{"dataset_name":"irregular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_2":{"dataset_name":"irregular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_echo_question":{"dataset_name":"left_branch_island_echo_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_echo_question","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_simple_question":{"dataset_name":"left_branch_island_simple_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_simple_question","unsafe_code":false,"validation_split":"train"},"blimp_matrix_question_npi_licensor_present":{"dataset_name":"matrix_question_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_matrix_question_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_1":{"dataset_name":"npi_present_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_1.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_1","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_2":{"dataset_name":"npi_present_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_2.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_2","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_licensor_present":{"dataset_name":"only_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_scope":{"dataset_name":"only_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_passive_1":{"dataset_name":"passive_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_1.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_1","unsafe_code":false,"validation_split":"train"},"blimp_passive_2":{"dataset_name":"passive_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_2.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_c_command":{"dataset_name":"principle_A_c_command","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_c_command.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_c_command","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_1":{"dataset_name":"principle_A_case_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_1.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_2":{"dataset_name":"principle_A_case_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_2.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_1":{"dataset_name":"principle_A_domain_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_1.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_2":{"dataset_name":"principle_A_domain_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_2.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_3":{"dataset_name":"principle_A_domain_3","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_3.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_3","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_reconstruction":{"dataset_name":"principle_A_reconstruction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_reconstruction.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_reconstruction","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_1":{"dataset_name":"regular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_2":{"dataset_name":"regular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_licensor_present":{"dataset_name":"sentential_negation_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_scope":{"dataset_name":"sentential_negation_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_sentential_subject_island":{"dataset_name":"sentential_subject_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_subject_island.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_subject_island","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_1":{"dataset_name":"superlative_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_2":{"dataset_name":"superlative_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_1":{"dataset_name":"tough_vs_raising_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_1.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_1","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_2":{"dataset_name":"tough_vs_raising_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_2.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_2","unsafe_code":false,"validation_split":"train"},"blimp_transitive":{"dataset_name":"transitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/transitive.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_transitive","unsafe_code":false,"validation_split":"train"},"blimp_wh_island":{"dataset_name":"wh_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_island.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_island","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_object_gap":{"dataset_name":"wh_questions_object_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_object_gap.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_object_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap":{"dataset_name":"wh_questions_subject_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap_long_distance":{"dataset_name":"wh_questions_subject_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap":{"dataset_name":"wh_vs_that_no_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap_long_distance":{"dataset_name":"wh_vs_that_no_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap":{"dataset_name":"wh_vs_that_with_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap_long_distance":{"dataset_name":"wh_vs_that_with_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap_long_distance","unsafe_code":false,"validation_split":"train"},"commonsense_qa_text":{"dataset_path":"tau/commonsense_qa","description":"","doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/config/tasks/commonsense_qa_text.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"commonsense_qa_text","training_split":"train","unsafe_code":false,"validation_split":"validation"},"hellaswag":{"dataset_path":"Rowan/hellaswag","description":"","doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_config":{"doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":"<function process_docs at 0x72645c9311c0>","sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","process_docs":"def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\\n    def _process_doc(doc):\\n        ctx = doc[\\"ctx_a\\"] + \\" \\" + doc[\\"ctx_b\\"].capitalize()\\n        out_doc = {\\n            \\"query\\": preprocess(doc[\\"activity_label\\"] + \\": \\" + ctx),\\n            \\"choices\\": [preprocess(ending) for ending in doc[\\"endings\\"]],\\n            \\"gold\\": int(doc[\\"label\\"]),\\n        }\\n        return out_doc\\n\\n    return dataset.map(_process_doc)\\n","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"hellaswag","training_split":"train","unsafe_code":false,"validation_split":"validation"},"piqa":{"dataset_path":"baber/piqa","description":"","doc_to_choice":"{{[sol1, sol2]}}","doc_to_decontamination_query":"goal","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{[sol1, sol2]}}","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/piqa/piqa.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"piqa","training_split":"train","unsafe_code":false,"validation_split":"validation"}},"task_versions":{"arc_challenge":1,"arc_easy":1,"blimp":"2.0","blimp_adjunct_island":1,"blimp_anaphor_gender_agreement":1,"blimp_anaphor_number_agreement":1,"blimp_animate_subject_passive":1,"blimp_animate_subject_trans":1,"blimp_causative":1,"blimp_complex_NP_island":1,"blimp_coordinate_structure_constraint_complex_left_branch":1,"blimp_coordinate_structure_constraint_object_extraction":1,"blimp_determiner_noun_agreement_1":1,"blimp_determiner_noun_agreement_2":1,"blimp_determiner_noun_agreement_irregular_1":1,"blimp_determiner_noun_agreement_irregular_2":1,"blimp_determiner_noun_agreement_with_adj_2":1,"blimp_determiner_noun_agreement_with_adj_irregular_1":1,"blimp_determiner_noun_agreement_with_adj_irregular_2":1,"blimp_determiner_noun_agreement_with_adjective_1":1,"blimp_distractor_agreement_relational_noun":1,"blimp_distractor_agreement_relative_clause":1,"blimp_drop_argument":1,"blimp_ellipsis_n_bar_1":1,"blimp_ellipsis_n_bar_2":1,"blimp_existential_there_object_raising":1,"blimp_existential_there_quantifiers_1":1,"blimp_existential_there_quantifiers_2":1,"blimp_existential_there_subject_raising":1,"blimp_expletive_it_object_raising":1,"blimp_inchoative":1,"blimp_intransitive":1,"blimp_irregular_past_participle_adjectives":1,"blimp_irregular_past_participle_verbs":1,"blimp_irregular_plural_subject_verb_agreement_1":1,"blimp_irregular_plural_subject_verb_agreement_2":1,"blimp_left_branch_island_echo_question":1,"blimp_left_branch_island_simple_question":1,"blimp_matrix_question_npi_licensor_present":1,"blimp_npi_present_1":1,"blimp_npi_present_2":1,"blimp_only_npi_licensor_present":1,"blimp_only_npi_scope":1,"blimp_passive_1":1,"blimp_passive_2":1,"blimp_principle_A_c_command":1,"blimp_principle_A_case_1":1,"blimp_principle_A_case_2":1,"blimp_principle_A_domain_1":1,"blimp_principle_A_domain_2":1,"blimp_principle_A_domain_3":1,"blimp_principle_A_reconstruction":1,"blimp_regular_plural_subject_verb_agreement_1":1,"blimp_regular_plural_subject_verb_agreement_2":1,"blimp_sentential_negation_npi_licensor_present":1,"blimp_sentential_negation_npi_scope":1,"blimp_sentential_subject_island":1,"blimp_superlative_quantifiers_1":1,"blimp_superlative_quantifiers_2":1,"blimp_tough_vs_raising_1":1,"blimp_tough_vs_raising_2":1,"blimp_transitive":1,"blimp_wh_island":1,"blimp_wh_questions_object_gap":1,"blimp_wh_questions_subject_gap":1,"blimp_wh_questions_subject_gap_long_distance":1,"blimp_wh_vs_that_no_gap":1,"blimp_wh_vs_that_no_gap_long_distance":1,"blimp_wh_vs_that_with_gap":1,"blimp_wh_vs_that_with_gap_long_distance":1,"commonsense_qa_text":"Yaml","hellaswag":1,"piqa":1}},"rank_range":[1,2],"release_date":"2024-09-15","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/qwen2-5-0-5b/20260716T154647Z-v1.1","source":"Qwen/Qwen2.5-0.5B","track":"base","updated_at":"2026-07-16T15:47:02.275688+00:00","vocab_size":151936},{"artifacts":{"result_file":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gemma-3-270m/20260716T154647Z-v1.1/results_v1.1_merged.json","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gemma-3-270m/20260716T154647Z-v1.1"},"benchmarks":{"arc_challenge":{"accuracy":0.27986348122866894,"chance":0.25,"ci":[0.5688282138794095,7.281001137656425],"classification":"reliably above chance","label":"ARC-Challenge","n_samples":1172,"score":3.9817974971558585},"arc_easy":{"accuracy":0.5728114478114478,"chance":0.25,"ci":[40.404040404040394,45.62289562289561],"classification":"reliably above chance","label":"ARC-Easy","n_samples":2376,"score":43.0415263748597},"blimp":{"accuracy":0.8213405423280423,"categories":{"Anaphor agreement":0.99,"Argument structure":0.7935555555555555,"Binding":0.7918571428571427,"Control / raising":0.8374,"Determiner-noun agreement":0.956,"Ellipsis":0.87,"Filler-gap":0.7927142857142858,"Irregular forms":0.9295,"Island effects":0.68875,"NPI licensing":0.668142857142857,"Quantifiers":0.7084999999999999,"Subject-verb agreement":0.8296666666666667},"category_count":12,"chance":0.5,"ci":[63.49301306216931,64.53917559523809],"classification":"reliably above chance","label":"BLiMP","n_samples":70000,"score":64.26810846560846},"commonsense_qa":{"accuracy":0.4258804258804259,"chance":0.2,"ci":[24.754299754299755,31.71580671580671],"classification":"reliably above chance","label":"CommonsenseQA","n_samples":1221,"score":28.235053235053236},"hellaswag":{"accuracy":0.4144592710615415,"chance":0.25,"ci":[20.65325632344155,23.29582420500564],"classification":"reliably above chance","label":"HellaSwag","n_samples":10042,"score":21.927902808205534},"piqa":{"accuracy":0.6849836779107725,"chance":0.5,"ci":[32.86180631120783,41.240478781284004],"classification":"reliably above chance","label":"PIQA","n_samples":1838,"score":36.99673558215451}},"complete":true,"composite":{"ci":[37.774028206553304,39.9138118144519],"classification":"reliably above chance","score":38.912790926162764},"id":"gemma-3-270m","name":"Gemma 3 270M","params_m":270,"protocol":{"bootstrap_replicates":2000,"bootstrap_seed":20260716,"harness":"lm-eval==0.4.12","harness_git_hash":null,"inference_config":{"batch_size":"32","batch_sizes":[],"bootstrap_iters":100000,"device":"cuda:0","fewshot_seed":1234,"gen_kwargs":{},"limit":null,"model":"hf","model_args":{"dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false},"model_dtype":"torch.bfloat16","model_num_parameters":268098176,"model_revision":"main","model_sha":"","numpy_seed":1234,"random_seed":1234,"tasks":["hellaswag","piqa","arc_easy","arc_challenge","commonsense_qa_text","blimp"],"torch_seed":1234,"use_cache":null,"v1_1_sources":{"rechecked_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gemma-3-270m/20260716T154341Z-v1.1/results_v1.1_merged.json","unchanged_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gemma-3-270m/20260716T154341Z-v1.1/results_v1.1_merged.json"}},"n_shot":{"arc_easy":0,"blimp":0,"blimp_adjunct_island":0,"blimp_anaphor_gender_agreement":0,"blimp_anaphor_number_agreement":0,"blimp_animate_subject_passive":0,"blimp_animate_subject_trans":0,"blimp_causative":0,"blimp_complex_NP_island":0,"blimp_coordinate_structure_constraint_complex_left_branch":0,"blimp_coordinate_structure_constraint_object_extraction":0,"blimp_determiner_noun_agreement_1":0,"blimp_determiner_noun_agreement_2":0,"blimp_determiner_noun_agreement_irregular_1":0,"blimp_determiner_noun_agreement_irregular_2":0,"blimp_determiner_noun_agreement_with_adj_2":0,"blimp_determiner_noun_agreement_with_adj_irregular_1":0,"blimp_determiner_noun_agreement_with_adj_irregular_2":0,"blimp_determiner_noun_agreement_with_adjective_1":0,"blimp_distractor_agreement_relational_noun":0,"blimp_distractor_agreement_relative_clause":0,"blimp_drop_argument":0,"blimp_ellipsis_n_bar_1":0,"blimp_ellipsis_n_bar_2":0,"blimp_existential_there_object_raising":0,"blimp_existential_there_quantifiers_1":0,"blimp_existential_there_quantifiers_2":0,"blimp_existential_there_subject_raising":0,"blimp_expletive_it_object_raising":0,"blimp_inchoative":0,"blimp_intransitive":0,"blimp_irregular_past_participle_adjectives":0,"blimp_irregular_past_participle_verbs":0,"blimp_irregular_plural_subject_verb_agreement_1":0,"blimp_irregular_plural_subject_verb_agreement_2":0,"blimp_left_branch_island_echo_question":0,"blimp_left_branch_island_simple_question":0,"blimp_matrix_question_npi_licensor_present":0,"blimp_npi_present_1":0,"blimp_npi_present_2":0,"blimp_only_npi_licensor_present":0,"blimp_only_npi_scope":0,"blimp_passive_1":0,"blimp_passive_2":0,"blimp_principle_A_c_command":0,"blimp_principle_A_case_1":0,"blimp_principle_A_case_2":0,"blimp_principle_A_domain_1":0,"blimp_principle_A_domain_2":0,"blimp_principle_A_domain_3":0,"blimp_principle_A_reconstruction":0,"blimp_regular_plural_subject_verb_agreement_1":0,"blimp_regular_plural_subject_verb_agreement_2":0,"blimp_sentential_negation_npi_licensor_present":0,"blimp_sentential_negation_npi_scope":0,"blimp_sentential_subject_island":0,"blimp_superlative_quantifiers_1":0,"blimp_superlative_quantifiers_2":0,"blimp_tough_vs_raising_1":0,"blimp_tough_vs_raising_2":0,"blimp_transitive":0,"blimp_wh_island":0,"blimp_wh_questions_object_gap":0,"blimp_wh_questions_subject_gap":0,"blimp_wh_questions_subject_gap_long_distance":0,"blimp_wh_vs_that_no_gap":0,"blimp_wh_vs_that_no_gap_long_distance":0,"blimp_wh_vs_that_with_gap":0,"blimp_wh_vs_that_with_gap_long_distance":0,"commonsense_qa":0,"hellaswag":0,"piqa":0},"suite":"slm-benchmark-v1.1","task_config":{"arc_challenge":{"dataset_name":"ARC-Challenge","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_challenge.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_challenge","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"arc_easy":{"dataset_name":"ARC-Easy","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_easy.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_easy","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"blimp_adjunct_island":{"dataset_name":"adjunct_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/adjunct_island.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_adjunct_island","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_gender_agreement":{"dataset_name":"anaphor_gender_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_gender_agreement","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_number_agreement":{"dataset_name":"anaphor_number_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_number_agreement.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_number_agreement","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_passive":{"dataset_name":"animate_subject_passive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_passive.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_passive","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_trans":{"dataset_name":"animate_subject_trans","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_trans.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_trans","unsafe_code":false,"validation_split":"train"},"blimp_causative":{"dataset_name":"causative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/causative.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_causative","unsafe_code":false,"validation_split":"train"},"blimp_complex_NP_island":{"dataset_name":"complex_NP_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/complex_NP_island.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_complex_NP_island","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_complex_left_branch":{"dataset_name":"coordinate_structure_constraint_complex_left_branch","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_complex_left_branch","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_object_extraction":{"dataset_name":"coordinate_structure_constraint_object_extraction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_object_extraction","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_1":{"dataset_name":"determiner_noun_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_2":{"dataset_name":"determiner_noun_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_1":{"dataset_name":"determiner_noun_agreement_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_2":{"dataset_name":"determiner_noun_agreement_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_2":{"dataset_name":"determiner_noun_agreement_with_adj_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_1":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_2":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adjective_1":{"dataset_name":"determiner_noun_agreement_with_adjective_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adjective_1","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relational_noun":{"dataset_name":"distractor_agreement_relational_noun","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relational_noun","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relative_clause":{"dataset_name":"distractor_agreement_relative_clause","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relative_clause","unsafe_code":false,"validation_split":"train"},"blimp_drop_argument":{"dataset_name":"drop_argument","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/drop_argument.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_drop_argument","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_1":{"dataset_name":"ellipsis_n_bar_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_1","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_2":{"dataset_name":"ellipsis_n_bar_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_object_raising":{"dataset_name":"existential_there_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_object_raising.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_1":{"dataset_name":"existential_there_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_2":{"dataset_name":"existential_there_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_subject_raising":{"dataset_name":"existential_there_subject_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_subject_raising.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_subject_raising","unsafe_code":false,"validation_split":"train"},"blimp_expletive_it_object_raising":{"dataset_name":"expletive_it_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/expletive_it_object_raising.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_expletive_it_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_inchoative":{"dataset_name":"inchoative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/inchoative.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_inchoative","unsafe_code":false,"validation_split":"train"},"blimp_intransitive":{"dataset_name":"intransitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/intransitive.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_intransitive","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_adjectives":{"dataset_name":"irregular_past_participle_adjectives","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_adjectives","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_verbs":{"dataset_name":"irregular_past_participle_verbs","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_verbs","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_1":{"dataset_name":"irregular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_2":{"dataset_name":"irregular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_echo_question":{"dataset_name":"left_branch_island_echo_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_echo_question","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_simple_question":{"dataset_name":"left_branch_island_simple_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_simple_question","unsafe_code":false,"validation_split":"train"},"blimp_matrix_question_npi_licensor_present":{"dataset_name":"matrix_question_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_matrix_question_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_1":{"dataset_name":"npi_present_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_1.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_1","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_2":{"dataset_name":"npi_present_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_2.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_2","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_licensor_present":{"dataset_name":"only_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_scope":{"dataset_name":"only_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_passive_1":{"dataset_name":"passive_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_1.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_1","unsafe_code":false,"validation_split":"train"},"blimp_passive_2":{"dataset_name":"passive_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_2.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_c_command":{"dataset_name":"principle_A_c_command","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_c_command.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_c_command","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_1":{"dataset_name":"principle_A_case_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_1.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_2":{"dataset_name":"principle_A_case_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_2.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_1":{"dataset_name":"principle_A_domain_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_1.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_2":{"dataset_name":"principle_A_domain_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_2.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_3":{"dataset_name":"principle_A_domain_3","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_3.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_3","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_reconstruction":{"dataset_name":"principle_A_reconstruction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_reconstruction.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_reconstruction","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_1":{"dataset_name":"regular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_2":{"dataset_name":"regular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_licensor_present":{"dataset_name":"sentential_negation_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_scope":{"dataset_name":"sentential_negation_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_sentential_subject_island":{"dataset_name":"sentential_subject_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_subject_island.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_subject_island","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_1":{"dataset_name":"superlative_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_2":{"dataset_name":"superlative_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_1":{"dataset_name":"tough_vs_raising_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_1.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_1","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_2":{"dataset_name":"tough_vs_raising_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_2.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_2","unsafe_code":false,"validation_split":"train"},"blimp_transitive":{"dataset_name":"transitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/transitive.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_transitive","unsafe_code":false,"validation_split":"train"},"blimp_wh_island":{"dataset_name":"wh_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_island.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_island","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_object_gap":{"dataset_name":"wh_questions_object_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_object_gap.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_object_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap":{"dataset_name":"wh_questions_subject_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap_long_distance":{"dataset_name":"wh_questions_subject_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap":{"dataset_name":"wh_vs_that_no_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap_long_distance":{"dataset_name":"wh_vs_that_no_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap":{"dataset_name":"wh_vs_that_with_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap_long_distance":{"dataset_name":"wh_vs_that_with_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap_long_distance","unsafe_code":false,"validation_split":"train"},"commonsense_qa_text":{"dataset_path":"tau/commonsense_qa","description":"","doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/config/tasks/commonsense_qa_text.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"commonsense_qa_text","training_split":"train","unsafe_code":false,"validation_split":"validation"},"hellaswag":{"dataset_path":"Rowan/hellaswag","description":"","doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_config":{"doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":"<function process_docs at 0x7ff024f928e0>","sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","process_docs":"def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\\n    def _process_doc(doc):\\n        ctx = doc[\\"ctx_a\\"] + \\" \\" + doc[\\"ctx_b\\"].capitalize()\\n        out_doc = {\\n            \\"query\\": preprocess(doc[\\"activity_label\\"] + \\": \\" + ctx),\\n            \\"choices\\": [preprocess(ending) for ending in doc[\\"endings\\"]],\\n            \\"gold\\": int(doc[\\"label\\"]),\\n        }\\n        return out_doc\\n\\n    return dataset.map(_process_doc)\\n","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"hellaswag","training_split":"train","unsafe_code":false,"validation_split":"validation"},"piqa":{"dataset_path":"baber/piqa","description":"","doc_to_choice":"{{[sol1, sol2]}}","doc_to_decontamination_query":"goal","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{[sol1, sol2]}}","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/piqa/piqa.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"piqa","training_split":"train","unsafe_code":false,"validation_split":"validation"}},"task_versions":{"arc_challenge":1,"arc_easy":1,"blimp":"2.0","blimp_adjunct_island":1,"blimp_anaphor_gender_agreement":1,"blimp_anaphor_number_agreement":1,"blimp_animate_subject_passive":1,"blimp_animate_subject_trans":1,"blimp_causative":1,"blimp_complex_NP_island":1,"blimp_coordinate_structure_constraint_complex_left_branch":1,"blimp_coordinate_structure_constraint_object_extraction":1,"blimp_determiner_noun_agreement_1":1,"blimp_determiner_noun_agreement_2":1,"blimp_determiner_noun_agreement_irregular_1":1,"blimp_determiner_noun_agreement_irregular_2":1,"blimp_determiner_noun_agreement_with_adj_2":1,"blimp_determiner_noun_agreement_with_adj_irregular_1":1,"blimp_determiner_noun_agreement_with_adj_irregular_2":1,"blimp_determiner_noun_agreement_with_adjective_1":1,"blimp_distractor_agreement_relational_noun":1,"blimp_distractor_agreement_relative_clause":1,"blimp_drop_argument":1,"blimp_ellipsis_n_bar_1":1,"blimp_ellipsis_n_bar_2":1,"blimp_existential_there_object_raising":1,"blimp_existential_there_quantifiers_1":1,"blimp_existential_there_quantifiers_2":1,"blimp_existential_there_subject_raising":1,"blimp_expletive_it_object_raising":1,"blimp_inchoative":1,"blimp_intransitive":1,"blimp_irregular_past_participle_adjectives":1,"blimp_irregular_past_participle_verbs":1,"blimp_irregular_plural_subject_verb_agreement_1":1,"blimp_irregular_plural_subject_verb_agreement_2":1,"blimp_left_branch_island_echo_question":1,"blimp_left_branch_island_simple_question":1,"blimp_matrix_question_npi_licensor_present":1,"blimp_npi_present_1":1,"blimp_npi_present_2":1,"blimp_only_npi_licensor_present":1,"blimp_only_npi_scope":1,"blimp_passive_1":1,"blimp_passive_2":1,"blimp_principle_A_c_command":1,"blimp_principle_A_case_1":1,"blimp_principle_A_case_2":1,"blimp_principle_A_domain_1":1,"blimp_principle_A_domain_2":1,"blimp_principle_A_domain_3":1,"blimp_principle_A_reconstruction":1,"blimp_regular_plural_subject_verb_agreement_1":1,"blimp_regular_plural_subject_verb_agreement_2":1,"blimp_sentential_negation_npi_licensor_present":1,"blimp_sentential_negation_npi_scope":1,"blimp_sentential_subject_island":1,"blimp_superlative_quantifiers_1":1,"blimp_superlative_quantifiers_2":1,"blimp_tough_vs_raising_1":1,"blimp_tough_vs_raising_2":1,"blimp_transitive":1,"blimp_wh_island":1,"blimp_wh_questions_object_gap":1,"blimp_wh_questions_subject_gap":1,"blimp_wh_questions_subject_gap_long_distance":1,"blimp_wh_vs_that_no_gap":1,"blimp_wh_vs_that_no_gap_long_distance":1,"blimp_wh_vs_that_with_gap":1,"blimp_wh_vs_that_with_gap_long_distance":1,"commonsense_qa_text":"Yaml","hellaswag":1,"piqa":1}},"rank_range":[3,3],"release_date":"2025-08-05","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gemma-3-270m/20260716T154647Z-v1.1","source":"google/gemma-3-270m","track":"base","updated_at":"2026-07-16T15:46:49.790812+00:00","vocab_size":262144},{"artifacts":{"result_file":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/smollm2-135m/20260716T154647Z-v1.1/results_v1.1_merged.json","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/smollm2-135m/20260716T154647Z-v1.1"},"benchmarks":{"arc_challenge":{"accuracy":0.29692832764505117,"chance":0.25,"ci":[2.6166097838452806,9.670079635949946],"classification":"reliably above chance","label":"ARC-Challenge","n_samples":1172,"score":6.2571103526734895},"arc_easy":{"accuracy":0.5871212121212122,"chance":0.25,"ci":[42.255892255892256,47.53086419753087],"classification":"reliably above chance","label":"ARC-Easy","n_samples":2376,"score":44.949494949494955},"blimp":{"accuracy":0.8136013227513228,"categories":{"Anaphor agreement":0.977,"Argument structure":0.8337777777777778,"Binding":0.7632857142857142,"Control / raising":0.8392000000000002,"Determiner-noun agreement":0.968,"Ellipsis":0.8745,"Filler-gap":0.7977142857142857,"Irregular forms":0.8380000000000001,"Island effects":0.6892499999999999,"NPI licensing":0.6485714285714286,"Quantifiers":0.67025,"Subject-verb agreement":0.8636666666666667},"category_count":12,"chance":0.5,"ci":[61.932142030423265,63.0667154431217],"classification":"reliably above chance","label":"BLiMP","n_samples":70000,"score":62.72026455026456},"commonsense_qa":{"accuracy":0.35544635544635544,"chance":0.2,"ci":[16.152231777231773,22.81173218673217],"classification":"reliably above chance","label":"CommonsenseQA","n_samples":1221,"score":19.43079443079443},"hellaswag":{"accuracy":0.43218482374029077,"chance":0.25,"ci":[23.016331408086035,25.592511451902016],"classification":"reliably above chance","label":"HellaSwag","n_samples":10042,"score":24.291309832038767},"piqa":{"accuracy":0.6817192600652884,"chance":0.5,"ci":[32.097388465723604,40.58759521218715],"classification":"reliably above chance","label":"PIQA","n_samples":1838,"score":36.34385201305768}},"complete":true,"composite":{"ci":[36.68113506339092,38.849860312348774],"classification":"reliably above chance","score":37.843491883620345},"id":"smollm2-135m","name":"SmolLM2 135M","params_m":135,"protocol":{"bootstrap_replicates":2000,"bootstrap_seed":20260716,"harness":"lm-eval==0.4.12","harness_git_hash":null,"inference_config":{"batch_size":"auto","batch_sizes":[64],"bootstrap_iters":100000,"device":"cuda:0","fewshot_seed":1234,"gen_kwargs":{},"limit":null,"model":"hf","model_args":{"dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false},"model_dtype":"torch.bfloat16","model_num_parameters":134515008,"model_revision":"main","model_sha":"","numpy_seed":1234,"random_seed":1234,"tasks":["hellaswag","piqa","arc_easy","arc_challenge","commonsense_qa_text","blimp"],"torch_seed":1234,"use_cache":null,"v1_1_sources":{"rechecked_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/smollm2-135m/20260716T154341Z-v1.1/results_v1.1_merged.json","unchanged_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/smollm2-135m/20260716T154341Z-v1.1/results_v1.1_merged.json"}},"n_shot":{"arc_easy":0,"blimp":0,"blimp_adjunct_island":0,"blimp_anaphor_gender_agreement":0,"blimp_anaphor_number_agreement":0,"blimp_animate_subject_passive":0,"blimp_animate_subject_trans":0,"blimp_causative":0,"blimp_complex_NP_island":0,"blimp_coordinate_structure_constraint_complex_left_branch":0,"blimp_coordinate_structure_constraint_object_extraction":0,"blimp_determiner_noun_agreement_1":0,"blimp_determiner_noun_agreement_2":0,"blimp_determiner_noun_agreement_irregular_1":0,"blimp_determiner_noun_agreement_irregular_2":0,"blimp_determiner_noun_agreement_with_adj_2":0,"blimp_determiner_noun_agreement_with_adj_irregular_1":0,"blimp_determiner_noun_agreement_with_adj_irregular_2":0,"blimp_determiner_noun_agreement_with_adjective_1":0,"blimp_distractor_agreement_relational_noun":0,"blimp_distractor_agreement_relative_clause":0,"blimp_drop_argument":0,"blimp_ellipsis_n_bar_1":0,"blimp_ellipsis_n_bar_2":0,"blimp_existential_there_object_raising":0,"blimp_existential_there_quantifiers_1":0,"blimp_existential_there_quantifiers_2":0,"blimp_existential_there_subject_raising":0,"blimp_expletive_it_object_raising":0,"blimp_inchoative":0,"blimp_intransitive":0,"blimp_irregular_past_participle_adjectives":0,"blimp_irregular_past_participle_verbs":0,"blimp_irregular_plural_subject_verb_agreement_1":0,"blimp_irregular_plural_subject_verb_agreement_2":0,"blimp_left_branch_island_echo_question":0,"blimp_left_branch_island_simple_question":0,"blimp_matrix_question_npi_licensor_present":0,"blimp_npi_present_1":0,"blimp_npi_present_2":0,"blimp_only_npi_licensor_present":0,"blimp_only_npi_scope":0,"blimp_passive_1":0,"blimp_passive_2":0,"blimp_principle_A_c_command":0,"blimp_principle_A_case_1":0,"blimp_principle_A_case_2":0,"blimp_principle_A_domain_1":0,"blimp_principle_A_domain_2":0,"blimp_principle_A_domain_3":0,"blimp_principle_A_reconstruction":0,"blimp_regular_plural_subject_verb_agreement_1":0,"blimp_regular_plural_subject_verb_agreement_2":0,"blimp_sentential_negation_npi_licensor_present":0,"blimp_sentential_negation_npi_scope":0,"blimp_sentential_subject_island":0,"blimp_superlative_quantifiers_1":0,"blimp_superlative_quantifiers_2":0,"blimp_tough_vs_raising_1":0,"blimp_tough_vs_raising_2":0,"blimp_transitive":0,"blimp_wh_island":0,"blimp_wh_questions_object_gap":0,"blimp_wh_questions_subject_gap":0,"blimp_wh_questions_subject_gap_long_distance":0,"blimp_wh_vs_that_no_gap":0,"blimp_wh_vs_that_no_gap_long_distance":0,"blimp_wh_vs_that_with_gap":0,"blimp_wh_vs_that_with_gap_long_distance":0,"commonsense_qa":0,"hellaswag":0,"piqa":0},"suite":"slm-benchmark-v1.1","task_config":{"arc_challenge":{"dataset_name":"ARC-Challenge","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_challenge.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_challenge","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"arc_easy":{"dataset_name":"ARC-Easy","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_easy.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_easy","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"blimp_adjunct_island":{"dataset_name":"adjunct_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/adjunct_island.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_adjunct_island","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_gender_agreement":{"dataset_name":"anaphor_gender_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_gender_agreement","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_number_agreement":{"dataset_name":"anaphor_number_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_number_agreement.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_number_agreement","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_passive":{"dataset_name":"animate_subject_passive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_passive.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_passive","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_trans":{"dataset_name":"animate_subject_trans","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_trans.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_trans","unsafe_code":false,"validation_split":"train"},"blimp_causative":{"dataset_name":"causative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/causative.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_causative","unsafe_code":false,"validation_split":"train"},"blimp_complex_NP_island":{"dataset_name":"complex_NP_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/complex_NP_island.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_complex_NP_island","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_complex_left_branch":{"dataset_name":"coordinate_structure_constraint_complex_left_branch","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_complex_left_branch","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_object_extraction":{"dataset_name":"coordinate_structure_constraint_object_extraction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_object_extraction","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_1":{"dataset_name":"determiner_noun_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_2":{"dataset_name":"determiner_noun_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_1":{"dataset_name":"determiner_noun_agreement_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_2":{"dataset_name":"determiner_noun_agreement_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_2":{"dataset_name":"determiner_noun_agreement_with_adj_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_1":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_2":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adjective_1":{"dataset_name":"determiner_noun_agreement_with_adjective_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adjective_1","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relational_noun":{"dataset_name":"distractor_agreement_relational_noun","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relational_noun","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relative_clause":{"dataset_name":"distractor_agreement_relative_clause","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relative_clause","unsafe_code":false,"validation_split":"train"},"blimp_drop_argument":{"dataset_name":"drop_argument","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/drop_argument.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_drop_argument","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_1":{"dataset_name":"ellipsis_n_bar_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_1","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_2":{"dataset_name":"ellipsis_n_bar_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_object_raising":{"dataset_name":"existential_there_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_object_raising.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_1":{"dataset_name":"existential_there_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_2":{"dataset_name":"existential_there_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_subject_raising":{"dataset_name":"existential_there_subject_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_subject_raising.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_subject_raising","unsafe_code":false,"validation_split":"train"},"blimp_expletive_it_object_raising":{"dataset_name":"expletive_it_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/expletive_it_object_raising.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_expletive_it_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_inchoative":{"dataset_name":"inchoative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/inchoative.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_inchoative","unsafe_code":false,"validation_split":"train"},"blimp_intransitive":{"dataset_name":"intransitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/intransitive.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_intransitive","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_adjectives":{"dataset_name":"irregular_past_participle_adjectives","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_adjectives","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_verbs":{"dataset_name":"irregular_past_participle_verbs","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_verbs","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_1":{"dataset_name":"irregular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_2":{"dataset_name":"irregular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_echo_question":{"dataset_name":"left_branch_island_echo_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_echo_question","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_simple_question":{"dataset_name":"left_branch_island_simple_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_simple_question","unsafe_code":false,"validation_split":"train"},"blimp_matrix_question_npi_licensor_present":{"dataset_name":"matrix_question_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_matrix_question_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_1":{"dataset_name":"npi_present_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_1.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_1","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_2":{"dataset_name":"npi_present_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_2.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_2","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_licensor_present":{"dataset_name":"only_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_scope":{"dataset_name":"only_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_passive_1":{"dataset_name":"passive_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_1.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_1","unsafe_code":false,"validation_split":"train"},"blimp_passive_2":{"dataset_name":"passive_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_2.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_c_command":{"dataset_name":"principle_A_c_command","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_c_command.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_c_command","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_1":{"dataset_name":"principle_A_case_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_1.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_2":{"dataset_name":"principle_A_case_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_2.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_1":{"dataset_name":"principle_A_domain_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_1.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_2":{"dataset_name":"principle_A_domain_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_2.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_3":{"dataset_name":"principle_A_domain_3","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_3.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_3","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_reconstruction":{"dataset_name":"principle_A_reconstruction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_reconstruction.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_reconstruction","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_1":{"dataset_name":"regular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_2":{"dataset_name":"regular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_licensor_present":{"dataset_name":"sentential_negation_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_scope":{"dataset_name":"sentential_negation_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_sentential_subject_island":{"dataset_name":"sentential_subject_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_subject_island.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_subject_island","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_1":{"dataset_name":"superlative_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_2":{"dataset_name":"superlative_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_1":{"dataset_name":"tough_vs_raising_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_1.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_1","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_2":{"dataset_name":"tough_vs_raising_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_2.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_2","unsafe_code":false,"validation_split":"train"},"blimp_transitive":{"dataset_name":"transitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/transitive.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_transitive","unsafe_code":false,"validation_split":"train"},"blimp_wh_island":{"dataset_name":"wh_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_island.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_island","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_object_gap":{"dataset_name":"wh_questions_object_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_object_gap.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_object_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap":{"dataset_name":"wh_questions_subject_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap_long_distance":{"dataset_name":"wh_questions_subject_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap":{"dataset_name":"wh_vs_that_no_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap_long_distance":{"dataset_name":"wh_vs_that_no_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap":{"dataset_name":"wh_vs_that_with_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap_long_distance":{"dataset_name":"wh_vs_that_with_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap_long_distance","unsafe_code":false,"validation_split":"train"},"commonsense_qa_text":{"dataset_path":"tau/commonsense_qa","description":"","doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/config/tasks/commonsense_qa_text.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"commonsense_qa_text","training_split":"train","unsafe_code":false,"validation_split":"validation"},"hellaswag":{"dataset_path":"Rowan/hellaswag","description":"","doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_config":{"doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":"<function process_docs at 0x7162a1fca3e0>","sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","process_docs":"def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\\n    def _process_doc(doc):\\n        ctx = doc[\\"ctx_a\\"] + \\" \\" + doc[\\"ctx_b\\"].capitalize()\\n        out_doc = {\\n            \\"query\\": preprocess(doc[\\"activity_label\\"] + \\": \\" + ctx),\\n            \\"choices\\": [preprocess(ending) for ending in doc[\\"endings\\"]],\\n            \\"gold\\": int(doc[\\"label\\"]),\\n        }\\n        return out_doc\\n\\n    return dataset.map(_process_doc)\\n","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"hellaswag","training_split":"train","unsafe_code":false,"validation_split":"validation"},"piqa":{"dataset_path":"baber/piqa","description":"","doc_to_choice":"{{[sol1, sol2]}}","doc_to_decontamination_query":"goal","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{[sol1, sol2]}}","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/piqa/piqa.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"piqa","training_split":"train","unsafe_code":false,"validation_split":"validation"}},"task_versions":{"arc_challenge":1,"arc_easy":1,"blimp":"2.0","blimp_adjunct_island":1,"blimp_anaphor_gender_agreement":1,"blimp_anaphor_number_agreement":1,"blimp_animate_subject_passive":1,"blimp_animate_subject_trans":1,"blimp_causative":1,"blimp_complex_NP_island":1,"blimp_coordinate_structure_constraint_complex_left_branch":1,"blimp_coordinate_structure_constraint_object_extraction":1,"blimp_determiner_noun_agreement_1":1,"blimp_determiner_noun_agreement_2":1,"blimp_determiner_noun_agreement_irregular_1":1,"blimp_determiner_noun_agreement_irregular_2":1,"blimp_determiner_noun_agreement_with_adj_2":1,"blimp_determiner_noun_agreement_with_adj_irregular_1":1,"blimp_determiner_noun_agreement_with_adj_irregular_2":1,"blimp_determiner_noun_agreement_with_adjective_1":1,"blimp_distractor_agreement_relational_noun":1,"blimp_distractor_agreement_relative_clause":1,"blimp_drop_argument":1,"blimp_ellipsis_n_bar_1":1,"blimp_ellipsis_n_bar_2":1,"blimp_existential_there_object_raising":1,"blimp_existential_there_quantifiers_1":1,"blimp_existential_there_quantifiers_2":1,"blimp_existential_there_subject_raising":1,"blimp_expletive_it_object_raising":1,"blimp_inchoative":1,"blimp_intransitive":1,"blimp_irregular_past_participle_adjectives":1,"blimp_irregular_past_participle_verbs":1,"blimp_irregular_plural_subject_verb_agreement_1":1,"blimp_irregular_plural_subject_verb_agreement_2":1,"blimp_left_branch_island_echo_question":1,"blimp_left_branch_island_simple_question":1,"blimp_matrix_question_npi_licensor_present":1,"blimp_npi_present_1":1,"blimp_npi_present_2":1,"blimp_only_npi_licensor_present":1,"blimp_only_npi_scope":1,"blimp_passive_1":1,"blimp_passive_2":1,"blimp_principle_A_c_command":1,"blimp_principle_A_case_1":1,"blimp_principle_A_case_2":1,"blimp_principle_A_domain_1":1,"blimp_principle_A_domain_2":1,"blimp_principle_A_domain_3":1,"blimp_principle_A_reconstruction":1,"blimp_regular_plural_subject_verb_agreement_1":1,"blimp_regular_plural_subject_verb_agreement_2":1,"blimp_sentential_negation_npi_licensor_present":1,"blimp_sentential_negation_npi_scope":1,"blimp_sentential_subject_island":1,"blimp_superlative_quantifiers_1":1,"blimp_superlative_quantifiers_2":1,"blimp_tough_vs_raising_1":1,"blimp_tough_vs_raising_2":1,"blimp_transitive":1,"blimp_wh_island":1,"blimp_wh_questions_object_gap":1,"blimp_wh_questions_subject_gap":1,"blimp_wh_questions_subject_gap_long_distance":1,"blimp_wh_vs_that_no_gap":1,"blimp_wh_vs_that_no_gap_long_distance":1,"blimp_wh_vs_that_with_gap":1,"blimp_wh_vs_that_with_gap_long_distance":1,"commonsense_qa_text":"Yaml","hellaswag":1,"piqa":1}},"rank_range":[4,4],"release_date":"2024-10-31","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/smollm2-135m/20260716T154647Z-v1.1","source":"HuggingFaceTB/SmolLM2-135M","track":"base","updated_at":"2026-07-16T15:47:05.346313+00:00","vocab_size":49152},{"artifacts":{"result_file":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/limen-0-2b/20260727T165752Z/models__limen-0-2b/results_2026-07-27T19-06-16.084476.json","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/limen-0-2b/20260727T165752Z"},"benchmarks":{"arc_challenge":{"accuracy":0.29692832764505117,"chance":0.25,"ci":[2.730375426621164,9.897610921501705],"classification":"reliably above chance","label":"ARC-Challenge","n_samples":1172,"score":6.2571103526734895},"arc_easy":{"accuracy":0.5336700336700336,"chance":0.25,"ci":[35.18518518518518,40.404040404040394],"classification":"reliably above chance","label":"ARC-Easy","n_samples":2376,"score":37.82267115600448},"blimp":{"accuracy":0.8327736772486772,"categories":{"Anaphor agreement":0.993,"Argument structure":0.8152222222222222,"Binding":0.7435714285714285,"Control / raising":0.8097999999999999,"Determiner-noun agreement":0.954125,"Ellipsis":0.862,"Filler-gap":0.7794285714285715,"Irregular forms":0.905,"Island effects":0.704375,"NPI licensing":0.8164285714285714,"Quantifiers":0.722,"Subject-verb agreement":0.8883333333333332},"category_count":12,"chance":0.5,"ci":[65.83977000661378,66.87867873677249],"classification":"reliably above chance","label":"BLiMP","n_samples":70000,"score":66.55473544973545},"commonsense_qa":{"accuracy":0.3415233415233415,"chance":0.2,"ci":[14.312039312039309,21.068796068796065],"classification":"reliably above chance","label":"CommonsenseQA","n_samples":1221,"score":17.690417690417686},"hellaswag":{"accuracy":0.41983668591913964,"chance":0.25,"ci":[21.33041226847242,23.9328155081989],"classification":"reliably above chance","label":"HellaSwag","n_samples":10042,"score":22.644891455885286},"piqa":{"accuracy":0.6735582154515778,"chance":0.5,"ci":[30.359085963003274,38.84929270946679],"classification":"reliably above chance","label":"PIQA","n_samples":1838,"score":34.71164309031556}},"complete":true,"composite":{"ci":[35.46080088232033,37.604706072519605],"classification":"reliably above chance","score":36.62932947399655},"id":"limen-0-2b","name":"Limen0.2B","params_m":222.51648,"protocol":{"bootstrap_replicates":2000,"bootstrap_seed":20260716,"harness":"lm-eval==0.4.12","harness_git_hash":"dc3c1cf","inference_config":{"batch_size":"64","batch_sizes":[],"bootstrap_iters":100000,"device":"cuda:0","fewshot_seed":1234,"gen_kwargs":{},"limit":null,"model":"hf","model_args":{"dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true},"model_dtype":"torch.float32","model_num_parameters":222516480,"model_revision":"main","model_sha":"","numpy_seed":1234,"random_seed":1234,"torch_seed":1234,"use_cache":null},"n_shot":{"arc_challenge":0,"arc_easy":0,"blimp":0,"blimp_adjunct_island":0,"blimp_anaphor_gender_agreement":0,"blimp_anaphor_number_agreement":0,"blimp_animate_subject_passive":0,"blimp_animate_subject_trans":0,"blimp_causative":0,"blimp_complex_NP_island":0,"blimp_coordinate_structure_constraint_complex_left_branch":0,"blimp_coordinate_structure_constraint_object_extraction":0,"blimp_determiner_noun_agreement_1":0,"blimp_determiner_noun_agreement_2":0,"blimp_determiner_noun_agreement_irregular_1":0,"blimp_determiner_noun_agreement_irregular_2":0,"blimp_determiner_noun_agreement_with_adj_2":0,"blimp_determiner_noun_agreement_with_adj_irregular_1":0,"blimp_determiner_noun_agreement_with_adj_irregular_2":0,"blimp_determiner_noun_agreement_with_adjective_1":0,"blimp_distractor_agreement_relational_noun":0,"blimp_distractor_agreement_relative_clause":0,"blimp_drop_argument":0,"blimp_ellipsis_n_bar_1":0,"blimp_ellipsis_n_bar_2":0,"blimp_existential_there_object_raising":0,"blimp_existential_there_quantifiers_1":0,"blimp_existential_there_quantifiers_2":0,"blimp_existential_there_subject_raising":0,"blimp_expletive_it_object_raising":0,"blimp_inchoative":0,"blimp_intransitive":0,"blimp_irregular_past_participle_adjectives":0,"blimp_irregular_past_participle_verbs":0,"blimp_irregular_plural_subject_verb_agreement_1":0,"blimp_irregular_plural_subject_verb_agreement_2":0,"blimp_left_branch_island_echo_question":0,"blimp_left_branch_island_simple_question":0,"blimp_matrix_question_npi_licensor_present":0,"blimp_npi_present_1":0,"blimp_npi_present_2":0,"blimp_only_npi_licensor_present":0,"blimp_only_npi_scope":0,"blimp_passive_1":0,"blimp_passive_2":0,"blimp_principle_A_c_command":0,"blimp_principle_A_case_1":0,"blimp_principle_A_case_2":0,"blimp_principle_A_domain_1":0,"blimp_principle_A_domain_2":0,"blimp_principle_A_domain_3":0,"blimp_principle_A_reconstruction":0,"blimp_regular_plural_subject_verb_agreement_1":0,"blimp_regular_plural_subject_verb_agreement_2":0,"blimp_sentential_negation_npi_licensor_present":0,"blimp_sentential_negation_npi_scope":0,"blimp_sentential_subject_island":0,"blimp_superlative_quantifiers_1":0,"blimp_superlative_quantifiers_2":0,"blimp_tough_vs_raising_1":0,"blimp_tough_vs_raising_2":0,"blimp_transitive":0,"blimp_wh_island":0,"blimp_wh_questions_object_gap":0,"blimp_wh_questions_subject_gap":0,"blimp_wh_questions_subject_gap_long_distance":0,"blimp_wh_vs_that_no_gap":0,"blimp_wh_vs_that_no_gap_long_distance":0,"blimp_wh_vs_that_with_gap":0,"blimp_wh_vs_that_with_gap_long_distance":0,"commonsense_qa_text":0,"hellaswag":0,"piqa":0},"suite":"slm-benchmark-v1.1","task_config":{"arc_challenge":{"dataset_name":"ARC-Challenge","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_challenge.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_challenge","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"arc_easy":{"dataset_name":"ARC-Easy","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_easy.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_easy","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"blimp_adjunct_island":{"dataset_name":"adjunct_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/adjunct_island.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_adjunct_island","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_gender_agreement":{"dataset_name":"anaphor_gender_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_gender_agreement","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_number_agreement":{"dataset_name":"anaphor_number_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_number_agreement.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_number_agreement","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_passive":{"dataset_name":"animate_subject_passive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_passive.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_passive","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_trans":{"dataset_name":"animate_subject_trans","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_trans.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_trans","unsafe_code":false,"validation_split":"train"},"blimp_causative":{"dataset_name":"causative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/causative.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_causative","unsafe_code":false,"validation_split":"train"},"blimp_complex_NP_island":{"dataset_name":"complex_NP_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/complex_NP_island.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_complex_NP_island","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_complex_left_branch":{"dataset_name":"coordinate_structure_constraint_complex_left_branch","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_complex_left_branch","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_object_extraction":{"dataset_name":"coordinate_structure_constraint_object_extraction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_object_extraction","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_1":{"dataset_name":"determiner_noun_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_2":{"dataset_name":"determiner_noun_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_1":{"dataset_name":"determiner_noun_agreement_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_2":{"dataset_name":"determiner_noun_agreement_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_2":{"dataset_name":"determiner_noun_agreement_with_adj_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_1":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_2":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adjective_1":{"dataset_name":"determiner_noun_agreement_with_adjective_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adjective_1","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relational_noun":{"dataset_name":"distractor_agreement_relational_noun","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relational_noun","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relative_clause":{"dataset_name":"distractor_agreement_relative_clause","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relative_clause","unsafe_code":false,"validation_split":"train"},"blimp_drop_argument":{"dataset_name":"drop_argument","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/drop_argument.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_drop_argument","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_1":{"dataset_name":"ellipsis_n_bar_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_1","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_2":{"dataset_name":"ellipsis_n_bar_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_object_raising":{"dataset_name":"existential_there_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_object_raising.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_1":{"dataset_name":"existential_there_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_2":{"dataset_name":"existential_there_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_subject_raising":{"dataset_name":"existential_there_subject_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_subject_raising.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_subject_raising","unsafe_code":false,"validation_split":"train"},"blimp_expletive_it_object_raising":{"dataset_name":"expletive_it_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/expletive_it_object_raising.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_expletive_it_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_inchoative":{"dataset_name":"inchoative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/inchoative.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_inchoative","unsafe_code":false,"validation_split":"train"},"blimp_intransitive":{"dataset_name":"intransitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/intransitive.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_intransitive","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_adjectives":{"dataset_name":"irregular_past_participle_adjectives","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_adjectives","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_verbs":{"dataset_name":"irregular_past_participle_verbs","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_verbs","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_1":{"dataset_name":"irregular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_2":{"dataset_name":"irregular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_echo_question":{"dataset_name":"left_branch_island_echo_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_echo_question","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_simple_question":{"dataset_name":"left_branch_island_simple_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_simple_question","unsafe_code":false,"validation_split":"train"},"blimp_matrix_question_npi_licensor_present":{"dataset_name":"matrix_question_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_matrix_question_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_1":{"dataset_name":"npi_present_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_1.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_1","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_2":{"dataset_name":"npi_present_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_2.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_2","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_licensor_present":{"dataset_name":"only_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_licensor_present.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_scope":{"dataset_name":"only_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_scope.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_passive_1":{"dataset_name":"passive_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_1.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_1","unsafe_code":false,"validation_split":"train"},"blimp_passive_2":{"dataset_name":"passive_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_2.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_c_command":{"dataset_name":"principle_A_c_command","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_c_command.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_c_command","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_1":{"dataset_name":"principle_A_case_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_1.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_2":{"dataset_name":"principle_A_case_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_2.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_1":{"dataset_name":"principle_A_domain_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_1.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_2":{"dataset_name":"principle_A_domain_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_2.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_3":{"dataset_name":"principle_A_domain_3","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_3.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_3","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_reconstruction":{"dataset_name":"principle_A_reconstruction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_reconstruction.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_reconstruction","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_1":{"dataset_name":"regular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_2":{"dataset_name":"regular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_licensor_present":{"dataset_name":"sentential_negation_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_scope":{"dataset_name":"sentential_negation_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_sentential_subject_island":{"dataset_name":"sentential_subject_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_subject_island.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_subject_island","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_1":{"dataset_name":"superlative_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_2":{"dataset_name":"superlative_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_1":{"dataset_name":"tough_vs_raising_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_1.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_1","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_2":{"dataset_name":"tough_vs_raising_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_2.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_2","unsafe_code":false,"validation_split":"train"},"blimp_transitive":{"dataset_name":"transitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/transitive.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_transitive","unsafe_code":false,"validation_split":"train"},"blimp_wh_island":{"dataset_name":"wh_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_island.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_island","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_object_gap":{"dataset_name":"wh_questions_object_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_object_gap.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_object_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap":{"dataset_name":"wh_questions_subject_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap_long_distance":{"dataset_name":"wh_questions_subject_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap":{"dataset_name":"wh_vs_that_no_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap_long_distance":{"dataset_name":"wh_vs_that_no_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap":{"dataset_name":"wh_vs_that_with_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap_long_distance":{"dataset_name":"wh_vs_that_with_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap_long_distance","unsafe_code":false,"validation_split":"train"},"commonsense_qa_text":{"dataset_path":"tau/commonsense_qa","description":"","doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/config/tasks/commonsense_qa_text.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"commonsense_qa_text","training_split":"train","unsafe_code":false,"validation_split":"validation"},"hellaswag":{"dataset_path":"Rowan/hellaswag","description":"","doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_config":{"doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":"<function process_docs at 0x7e1a5f77d120>","sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","process_docs":"def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\\n    def _process_doc(doc):\\n        ctx = doc[\\"ctx_a\\"] + \\" \\" + doc[\\"ctx_b\\"].capitalize()\\n        out_doc = {\\n            \\"query\\": preprocess(doc[\\"activity_label\\"] + \\": \\" + ctx),\\n            \\"choices\\": [preprocess(ending) for ending in doc[\\"endings\\"]],\\n            \\"gold\\": int(doc[\\"label\\"]),\\n        }\\n        return out_doc\\n\\n    return dataset.map(_process_doc)\\n","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"hellaswag","training_split":"train","unsafe_code":false,"validation_split":"validation"},"piqa":{"dataset_path":"baber/piqa","description":"","doc_to_choice":"{{[sol1, sol2]}}","doc_to_decontamination_query":"goal","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{[sol1, sol2]}}","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/piqa/piqa.yaml","dtype":"bfloat16","prefix_token_id":16383,"pretrained":"models/limen-0-2b","tokenizer":"models/limen-0-2b","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"piqa","training_split":"train","unsafe_code":false,"validation_split":"validation"}},"task_versions":{"arc_challenge":1,"arc_easy":1,"blimp":"2.0","blimp_adjunct_island":1,"blimp_anaphor_gender_agreement":1,"blimp_anaphor_number_agreement":1,"blimp_animate_subject_passive":1,"blimp_animate_subject_trans":1,"blimp_causative":1,"blimp_complex_NP_island":1,"blimp_coordinate_structure_constraint_complex_left_branch":1,"blimp_coordinate_structure_constraint_object_extraction":1,"blimp_determiner_noun_agreement_1":1,"blimp_determiner_noun_agreement_2":1,"blimp_determiner_noun_agreement_irregular_1":1,"blimp_determiner_noun_agreement_irregular_2":1,"blimp_determiner_noun_agreement_with_adj_2":1,"blimp_determiner_noun_agreement_with_adj_irregular_1":1,"blimp_determiner_noun_agreement_with_adj_irregular_2":1,"blimp_determiner_noun_agreement_with_adjective_1":1,"blimp_distractor_agreement_relational_noun":1,"blimp_distractor_agreement_relative_clause":1,"blimp_drop_argument":1,"blimp_ellipsis_n_bar_1":1,"blimp_ellipsis_n_bar_2":1,"blimp_existential_there_object_raising":1,"blimp_existential_there_quantifiers_1":1,"blimp_existential_there_quantifiers_2":1,"blimp_existential_there_subject_raising":1,"blimp_expletive_it_object_raising":1,"blimp_inchoative":1,"blimp_intransitive":1,"blimp_irregular_past_participle_adjectives":1,"blimp_irregular_past_participle_verbs":1,"blimp_irregular_plural_subject_verb_agreement_1":1,"blimp_irregular_plural_subject_verb_agreement_2":1,"blimp_left_branch_island_echo_question":1,"blimp_left_branch_island_simple_question":1,"blimp_matrix_question_npi_licensor_present":1,"blimp_npi_present_1":1,"blimp_npi_present_2":1,"blimp_only_npi_licensor_present":1,"blimp_only_npi_scope":1,"blimp_passive_1":1,"blimp_passive_2":1,"blimp_principle_A_c_command":1,"blimp_principle_A_case_1":1,"blimp_principle_A_case_2":1,"blimp_principle_A_domain_1":1,"blimp_principle_A_domain_2":1,"blimp_principle_A_domain_3":1,"blimp_principle_A_reconstruction":1,"blimp_regular_plural_subject_verb_agreement_1":1,"blimp_regular_plural_subject_verb_agreement_2":1,"blimp_sentential_negation_npi_licensor_present":1,"blimp_sentential_negation_npi_scope":1,"blimp_sentential_subject_island":1,"blimp_superlative_quantifiers_1":1,"blimp_superlative_quantifiers_2":1,"blimp_tough_vs_raising_1":1,"blimp_tough_vs_raising_2":1,"blimp_transitive":1,"blimp_wh_island":1,"blimp_wh_questions_object_gap":1,"blimp_wh_questions_subject_gap":1,"blimp_wh_questions_subject_gap_long_distance":1,"blimp_wh_vs_that_no_gap":1,"blimp_wh_vs_that_no_gap_long_distance":1,"blimp_wh_vs_that_with_gap":1,"blimp_wh_vs_that_with_gap_long_distance":1,"commonsense_qa_text":"Yaml","hellaswag":1,"piqa":1}},"rank_range":[5,5],"release_date":"2026-07-24","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/limen-0-2b/20260727T165752Z","source":"UniversalComputingResearch/Limen0.2B","track":"base","updated_at":"2026-07-27T17:06:22.066132+00:00","vocab_size":16384},{"artifacts":{"result_file":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt-x2-125m/20260716T154647Z-v1.1/results_v1.1_merged.json","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt-x2-125m/20260716T154647Z-v1.1"},"benchmarks":{"arc_challenge":{"accuracy":0.27559726962457337,"chance":0.25,"ci":[-0.11376564277588337,7.053469852104666],"classification":"indistinguishable from chance","label":"ARC-Challenge","n_samples":1172,"score":3.4129692832764498},"arc_easy":{"accuracy":0.5164141414141414,"chance":0.25,"ci":[32.82687991021324,38.21548821548822],"classification":"reliably above chance","label":"ARC-Easy","n_samples":2376,"score":35.52188552188553},"blimp":{"accuracy":0.8291201719576721,"categories":{"Anaphor agreement":0.9864999999999999,"Argument structure":0.826111111111111,"Binding":0.7557142857142857,"Control / raising":0.8231999999999999,"Determiner-noun agreement":0.9581249999999999,"Ellipsis":0.8534999999999999,"Filler-gap":0.7958571428571429,"Irregular forms":0.923,"Island effects":0.6868749999999999,"NPI licensing":0.7461428571428571,"Quantifiers":0.70725,"Subject-verb agreement":0.8871666666666665},"category_count":12,"chance":0.5,"ci":[64.96420122354498,66.0613258928571],"classification":"reliably above chance","label":"BLiMP","n_samples":70000,"score":65.82403439153441},"commonsense_qa":{"accuracy":0.3464373464373464,"chance":0.2,"ci":[14.926289926289922,21.580671580671574],"classification":"reliably above chance","label":"CommonsenseQA","n_samples":1221,"score":18.3046683046683},"hellaswag":{"accuracy":0.4051981676956781,"chance":0.25,"ci":[19.431388169687313,21.941512314943907],"classification":"reliably above chance","label":"HellaSwag","n_samples":10042,"score":20.693089026090416},"piqa":{"accuracy":0.6708378672470077,"chance":0.5,"ci":[29.706202393906423,38.41131664853101],"classification":"reliably above chance","label":"PIQA","n_samples":1838,"score":34.16757344940153}},"complete":true,"composite":{"ci":[34.34048401085747,36.481411360993164],"classification":"reliably above chance","score":35.52389451013529},"id":"gpt-x2-125m","name":"GPT-X2 125M","params_m":125.081664,"protocol":{"bootstrap_replicates":2000,"bootstrap_seed":20260716,"harness":"lm-eval==0.4.12","harness_git_hash":null,"inference_config":{"batch_size":"auto","batch_sizes":[64],"bootstrap_iters":100000,"device":"cuda:0","fewshot_seed":1234,"gen_kwargs":{},"limit":null,"model":"hf","model_args":{"dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true},"model_dtype":"torch.bfloat16","model_num_parameters":125081664,"model_revision":"main","model_sha":"","numpy_seed":1234,"random_seed":1234,"tasks":["hellaswag","piqa","arc_easy","arc_challenge","commonsense_qa_text","blimp"],"torch_seed":1234,"use_cache":null,"v1_1_sources":{"rechecked_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt-x2-125m/20260716T154341Z-v1.1/results_v1.1_merged.json","unchanged_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt-x2-125m/20260716T154341Z-v1.1/results_v1.1_merged.json"}},"n_shot":{"arc_easy":0,"blimp":0,"blimp_adjunct_island":0,"blimp_anaphor_gender_agreement":0,"blimp_anaphor_number_agreement":0,"blimp_animate_subject_passive":0,"blimp_animate_subject_trans":0,"blimp_causative":0,"blimp_complex_NP_island":0,"blimp_coordinate_structure_constraint_complex_left_branch":0,"blimp_coordinate_structure_constraint_object_extraction":0,"blimp_determiner_noun_agreement_1":0,"blimp_determiner_noun_agreement_2":0,"blimp_determiner_noun_agreement_irregular_1":0,"blimp_determiner_noun_agreement_irregular_2":0,"blimp_determiner_noun_agreement_with_adj_2":0,"blimp_determiner_noun_agreement_with_adj_irregular_1":0,"blimp_determiner_noun_agreement_with_adj_irregular_2":0,"blimp_determiner_noun_agreement_with_adjective_1":0,"blimp_distractor_agreement_relational_noun":0,"blimp_distractor_agreement_relative_clause":0,"blimp_drop_argument":0,"blimp_ellipsis_n_bar_1":0,"blimp_ellipsis_n_bar_2":0,"blimp_existential_there_object_raising":0,"blimp_existential_there_quantifiers_1":0,"blimp_existential_there_quantifiers_2":0,"blimp_existential_there_subject_raising":0,"blimp_expletive_it_object_raising":0,"blimp_inchoative":0,"blimp_intransitive":0,"blimp_irregular_past_participle_adjectives":0,"blimp_irregular_past_participle_verbs":0,"blimp_irregular_plural_subject_verb_agreement_1":0,"blimp_irregular_plural_subject_verb_agreement_2":0,"blimp_left_branch_island_echo_question":0,"blimp_left_branch_island_simple_question":0,"blimp_matrix_question_npi_licensor_present":0,"blimp_npi_present_1":0,"blimp_npi_present_2":0,"blimp_only_npi_licensor_present":0,"blimp_only_npi_scope":0,"blimp_passive_1":0,"blimp_passive_2":0,"blimp_principle_A_c_command":0,"blimp_principle_A_case_1":0,"blimp_principle_A_case_2":0,"blimp_principle_A_domain_1":0,"blimp_principle_A_domain_2":0,"blimp_principle_A_domain_3":0,"blimp_principle_A_reconstruction":0,"blimp_regular_plural_subject_verb_agreement_1":0,"blimp_regular_plural_subject_verb_agreement_2":0,"blimp_sentential_negation_npi_licensor_present":0,"blimp_sentential_negation_npi_scope":0,"blimp_sentential_subject_island":0,"blimp_superlative_quantifiers_1":0,"blimp_superlative_quantifiers_2":0,"blimp_tough_vs_raising_1":0,"blimp_tough_vs_raising_2":0,"blimp_transitive":0,"blimp_wh_island":0,"blimp_wh_questions_object_gap":0,"blimp_wh_questions_subject_gap":0,"blimp_wh_questions_subject_gap_long_distance":0,"blimp_wh_vs_that_no_gap":0,"blimp_wh_vs_that_no_gap_long_distance":0,"blimp_wh_vs_that_with_gap":0,"blimp_wh_vs_that_with_gap_long_distance":0,"commonsense_qa":0,"hellaswag":0,"piqa":0},"suite":"slm-benchmark-v1.1","task_config":{"arc_challenge":{"dataset_name":"ARC-Challenge","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_challenge.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_challenge","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"arc_easy":{"dataset_name":"ARC-Easy","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_easy.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_easy","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"blimp_adjunct_island":{"dataset_name":"adjunct_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/adjunct_island.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_adjunct_island","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_gender_agreement":{"dataset_name":"anaphor_gender_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_gender_agreement","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_number_agreement":{"dataset_name":"anaphor_number_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_number_agreement.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_number_agreement","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_passive":{"dataset_name":"animate_subject_passive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_passive.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_passive","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_trans":{"dataset_name":"animate_subject_trans","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_trans.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_trans","unsafe_code":false,"validation_split":"train"},"blimp_causative":{"dataset_name":"causative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/causative.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_causative","unsafe_code":false,"validation_split":"train"},"blimp_complex_NP_island":{"dataset_name":"complex_NP_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/complex_NP_island.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_complex_NP_island","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_complex_left_branch":{"dataset_name":"coordinate_structure_constraint_complex_left_branch","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_complex_left_branch","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_object_extraction":{"dataset_name":"coordinate_structure_constraint_object_extraction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_object_extraction","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_1":{"dataset_name":"determiner_noun_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_2":{"dataset_name":"determiner_noun_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_1":{"dataset_name":"determiner_noun_agreement_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_2":{"dataset_name":"determiner_noun_agreement_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_2":{"dataset_name":"determiner_noun_agreement_with_adj_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_1":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_2":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adjective_1":{"dataset_name":"determiner_noun_agreement_with_adjective_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adjective_1","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relational_noun":{"dataset_name":"distractor_agreement_relational_noun","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relational_noun","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relative_clause":{"dataset_name":"distractor_agreement_relative_clause","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relative_clause","unsafe_code":false,"validation_split":"train"},"blimp_drop_argument":{"dataset_name":"drop_argument","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/drop_argument.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_drop_argument","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_1":{"dataset_name":"ellipsis_n_bar_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_1","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_2":{"dataset_name":"ellipsis_n_bar_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_object_raising":{"dataset_name":"existential_there_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_object_raising.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_1":{"dataset_name":"existential_there_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_2":{"dataset_name":"existential_there_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_subject_raising":{"dataset_name":"existential_there_subject_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_subject_raising.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_subject_raising","unsafe_code":false,"validation_split":"train"},"blimp_expletive_it_object_raising":{"dataset_name":"expletive_it_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/expletive_it_object_raising.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_expletive_it_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_inchoative":{"dataset_name":"inchoative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/inchoative.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_inchoative","unsafe_code":false,"validation_split":"train"},"blimp_intransitive":{"dataset_name":"intransitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/intransitive.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_intransitive","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_adjectives":{"dataset_name":"irregular_past_participle_adjectives","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_adjectives","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_verbs":{"dataset_name":"irregular_past_participle_verbs","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_verbs","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_1":{"dataset_name":"irregular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_2":{"dataset_name":"irregular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_echo_question":{"dataset_name":"left_branch_island_echo_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_echo_question","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_simple_question":{"dataset_name":"left_branch_island_simple_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_simple_question","unsafe_code":false,"validation_split":"train"},"blimp_matrix_question_npi_licensor_present":{"dataset_name":"matrix_question_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_matrix_question_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_1":{"dataset_name":"npi_present_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_1","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_2":{"dataset_name":"npi_present_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_2","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_licensor_present":{"dataset_name":"only_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_scope":{"dataset_name":"only_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_passive_1":{"dataset_name":"passive_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_1","unsafe_code":false,"validation_split":"train"},"blimp_passive_2":{"dataset_name":"passive_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_c_command":{"dataset_name":"principle_A_c_command","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_c_command.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_c_command","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_1":{"dataset_name":"principle_A_case_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_2":{"dataset_name":"principle_A_case_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_1":{"dataset_name":"principle_A_domain_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_2":{"dataset_name":"principle_A_domain_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_3":{"dataset_name":"principle_A_domain_3","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_3.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_3","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_reconstruction":{"dataset_name":"principle_A_reconstruction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_reconstruction.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_reconstruction","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_1":{"dataset_name":"regular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_2":{"dataset_name":"regular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_licensor_present":{"dataset_name":"sentential_negation_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_scope":{"dataset_name":"sentential_negation_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_sentential_subject_island":{"dataset_name":"sentential_subject_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_subject_island.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_subject_island","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_1":{"dataset_name":"superlative_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_2":{"dataset_name":"superlative_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_1":{"dataset_name":"tough_vs_raising_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_1","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_2":{"dataset_name":"tough_vs_raising_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_2","unsafe_code":false,"validation_split":"train"},"blimp_transitive":{"dataset_name":"transitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/transitive.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_transitive","unsafe_code":false,"validation_split":"train"},"blimp_wh_island":{"dataset_name":"wh_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_island.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_island","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_object_gap":{"dataset_name":"wh_questions_object_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_object_gap.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_object_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap":{"dataset_name":"wh_questions_subject_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap_long_distance":{"dataset_name":"wh_questions_subject_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap":{"dataset_name":"wh_vs_that_no_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap_long_distance":{"dataset_name":"wh_vs_that_no_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap":{"dataset_name":"wh_vs_that_with_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap_long_distance":{"dataset_name":"wh_vs_that_with_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap_long_distance","unsafe_code":false,"validation_split":"train"},"commonsense_qa_text":{"dataset_path":"tau/commonsense_qa","description":"","doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/config/tasks/commonsense_qa_text.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"commonsense_qa_text","training_split":"train","unsafe_code":false,"validation_split":"validation"},"hellaswag":{"dataset_path":"Rowan/hellaswag","description":"","doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_config":{"doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":"<function process_docs at 0x73557ef813a0>","sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","process_docs":"def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\\n    def _process_doc(doc):\\n        ctx = doc[\\"ctx_a\\"] + \\" \\" + doc[\\"ctx_b\\"].capitalize()\\n        out_doc = {\\n            \\"query\\": preprocess(doc[\\"activity_label\\"] + \\": \\" + ctx),\\n            \\"choices\\": [preprocess(ending) for ending in doc[\\"endings\\"]],\\n            \\"gold\\": int(doc[\\"label\\"]),\\n        }\\n        return out_doc\\n\\n    return dataset.map(_process_doc)\\n","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"hellaswag","training_split":"train","unsafe_code":false,"validation_split":"validation"},"piqa":{"dataset_path":"baber/piqa","description":"","doc_to_choice":"{{[sol1, sol2]}}","doc_to_decontamination_query":"goal","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{[sol1, sol2]}}","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/piqa/piqa.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"piqa","training_split":"train","unsafe_code":false,"validation_split":"validation"}},"task_versions":{"arc_challenge":1,"arc_easy":1,"blimp":"2.0","blimp_adjunct_island":1,"blimp_anaphor_gender_agreement":1,"blimp_anaphor_number_agreement":1,"blimp_animate_subject_passive":1,"blimp_animate_subject_trans":1,"blimp_causative":1,"blimp_complex_NP_island":1,"blimp_coordinate_structure_constraint_complex_left_branch":1,"blimp_coordinate_structure_constraint_object_extraction":1,"blimp_determiner_noun_agreement_1":1,"blimp_determiner_noun_agreement_2":1,"blimp_determiner_noun_agreement_irregular_1":1,"blimp_determiner_noun_agreement_irregular_2":1,"blimp_determiner_noun_agreement_with_adj_2":1,"blimp_determiner_noun_agreement_with_adj_irregular_1":1,"blimp_determiner_noun_agreement_with_adj_irregular_2":1,"blimp_determiner_noun_agreement_with_adjective_1":1,"blimp_distractor_agreement_relational_noun":1,"blimp_distractor_agreement_relative_clause":1,"blimp_drop_argument":1,"blimp_ellipsis_n_bar_1":1,"blimp_ellipsis_n_bar_2":1,"blimp_existential_there_object_raising":1,"blimp_existential_there_quantifiers_1":1,"blimp_existential_there_quantifiers_2":1,"blimp_existential_there_subject_raising":1,"blimp_expletive_it_object_raising":1,"blimp_inchoative":1,"blimp_intransitive":1,"blimp_irregular_past_participle_adjectives":1,"blimp_irregular_past_participle_verbs":1,"blimp_irregular_plural_subject_verb_agreement_1":1,"blimp_irregular_plural_subject_verb_agreement_2":1,"blimp_left_branch_island_echo_question":1,"blimp_left_branch_island_simple_question":1,"blimp_matrix_question_npi_licensor_present":1,"blimp_npi_present_1":1,"blimp_npi_present_2":1,"blimp_only_npi_licensor_present":1,"blimp_only_npi_scope":1,"blimp_passive_1":1,"blimp_passive_2":1,"blimp_principle_A_c_command":1,"blimp_principle_A_case_1":1,"blimp_principle_A_case_2":1,"blimp_principle_A_domain_1":1,"blimp_principle_A_domain_2":1,"blimp_principle_A_domain_3":1,"blimp_principle_A_reconstruction":1,"blimp_regular_plural_subject_verb_agreement_1":1,"blimp_regular_plural_subject_verb_agreement_2":1,"blimp_sentential_negation_npi_licensor_present":1,"blimp_sentential_negation_npi_scope":1,"blimp_sentential_subject_island":1,"blimp_superlative_quantifiers_1":1,"blimp_superlative_quantifiers_2":1,"blimp_tough_vs_raising_1":1,"blimp_tough_vs_raising_2":1,"blimp_transitive":1,"blimp_wh_island":1,"blimp_wh_questions_object_gap":1,"blimp_wh_questions_subject_gap":1,"blimp_wh_questions_subject_gap_long_distance":1,"blimp_wh_vs_that_no_gap":1,"blimp_wh_vs_that_no_gap_long_distance":1,"blimp_wh_vs_that_with_gap":1,"blimp_wh_vs_that_with_gap_long_distance":1,"commonsense_qa_text":"Yaml","hellaswag":1,"piqa":1}},"rank_range":[6,6],"release_date":"2026-03-25","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt-x2-125m/20260716T154647Z-v1.1","source":"AxiomicLabs/GPT-X2-125M","track":"base","updated_at":"2026-07-16T15:46:54.567291+00:00","vocab_size":32768},{"artifacts":{"result_file":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt2-medium-355m/20260723T113123Z/models__gpt2-medium-355m/results_2026-07-23T13-38-50.898870.json","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt2-medium-355m/20260723T113123Z"},"benchmarks":{"arc_challenge":{"accuracy":0.25,"chance":0.25,"ci":[-3.074516496018204,3.2992036405005662],"classification":"indistinguishable from chance","label":"ARC-Challenge","n_samples":1172,"score":0},"arc_easy":{"accuracy":0.43476430976430974,"chance":0.25,"ci":[22.053872053872052,27.328843995510663],"classification":"reliably above chance","label":"ARC-Easy","n_samples":2376,"score":24.635241301907968},"blimp":{"accuracy":0.8522063492063493,"categories":{"Anaphor agreement":0.9964999999999999,"Argument structure":0.8430000000000001,"Binding":0.7965714285714285,"Control / raising":0.8360000000000001,"Determiner-noun agreement":0.9661249999999999,"Ellipsis":0.874,"Filler-gap":0.825,"Irregular forms":0.95,"Island effects":0.719875,"NPI licensing":0.7845714285714287,"Quantifiers":0.7424999999999999,"Subject-verb agreement":0.8923333333333333},"category_count":12,"chance":0.5,"ci":[69.78037599206348,70.79667377645502],"classification":"reliably above chance","label":"BLiMP","n_samples":70000,"score":70.44126984126986},"commonsense_qa":{"accuracy":0.31695331695331697,"chance":0.2,"ci":[11.545352170352167,17.997542997542997],"classification":"reliably above chance","label":"CommonsenseQA","n_samples":1221,"score":14.61916461916462},"hellaswag":{"accuracy":0.3933479386576379,"chance":0.25,"ci":[17.86463519883157,20.34820420898891],"classification":"reliably above chance","label":"HellaSwag","n_samples":10042,"score":19.113058487685052},"piqa":{"accuracy":0.6659412404787813,"chance":0.5,"ci":[28.944504896626768,37.54080522306855],"classification":"reliably above chance","label":"PIQA","n_samples":1838,"score":33.188248095756265}},"complete":true,"composite":{"ci":[32.30209572902444,34.41893633162846],"classification":"reliably above chance","score":33.3828370715262},"id":"gpt2-medium-355m","name":"GPT-2 Medium 355M","params_m":354.823168,"protocol":{"bootstrap_replicates":2000,"bootstrap_seed":20260716,"harness":"lm-eval==0.4.12","harness_git_hash":"dc3c1cf","inference_config":{"batch_size":"16","batch_sizes":[],"bootstrap_iters":100000,"device":"cuda:0","fewshot_seed":1234,"gen_kwargs":{},"limit":null,"model":"hf","model_args":{"dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false},"model_dtype":"torch.bfloat16","model_num_parameters":354823168,"model_revision":"main","model_sha":"","numpy_seed":1234,"random_seed":1234,"torch_seed":1234,"use_cache":null},"n_shot":{"arc_challenge":0,"arc_easy":0,"blimp":0,"blimp_adjunct_island":0,"blimp_anaphor_gender_agreement":0,"blimp_anaphor_number_agreement":0,"blimp_animate_subject_passive":0,"blimp_animate_subject_trans":0,"blimp_causative":0,"blimp_complex_NP_island":0,"blimp_coordinate_structure_constraint_complex_left_branch":0,"blimp_coordinate_structure_constraint_object_extraction":0,"blimp_determiner_noun_agreement_1":0,"blimp_determiner_noun_agreement_2":0,"blimp_determiner_noun_agreement_irregular_1":0,"blimp_determiner_noun_agreement_irregular_2":0,"blimp_determiner_noun_agreement_with_adj_2":0,"blimp_determiner_noun_agreement_with_adj_irregular_1":0,"blimp_determiner_noun_agreement_with_adj_irregular_2":0,"blimp_determiner_noun_agreement_with_adjective_1":0,"blimp_distractor_agreement_relational_noun":0,"blimp_distractor_agreement_relative_clause":0,"blimp_drop_argument":0,"blimp_ellipsis_n_bar_1":0,"blimp_ellipsis_n_bar_2":0,"blimp_existential_there_object_raising":0,"blimp_existential_there_quantifiers_1":0,"blimp_existential_there_quantifiers_2":0,"blimp_existential_there_subject_raising":0,"blimp_expletive_it_object_raising":0,"blimp_inchoative":0,"blimp_intransitive":0,"blimp_irregular_past_participle_adjectives":0,"blimp_irregular_past_participle_verbs":0,"blimp_irregular_plural_subject_verb_agreement_1":0,"blimp_irregular_plural_subject_verb_agreement_2":0,"blimp_left_branch_island_echo_question":0,"blimp_left_branch_island_simple_question":0,"blimp_matrix_question_npi_licensor_present":0,"blimp_npi_present_1":0,"blimp_npi_present_2":0,"blimp_only_npi_licensor_present":0,"blimp_only_npi_scope":0,"blimp_passive_1":0,"blimp_passive_2":0,"blimp_principle_A_c_command":0,"blimp_principle_A_case_1":0,"blimp_principle_A_case_2":0,"blimp_principle_A_domain_1":0,"blimp_principle_A_domain_2":0,"blimp_principle_A_domain_3":0,"blimp_principle_A_reconstruction":0,"blimp_regular_plural_subject_verb_agreement_1":0,"blimp_regular_plural_subject_verb_agreement_2":0,"blimp_sentential_negation_npi_licensor_present":0,"blimp_sentential_negation_npi_scope":0,"blimp_sentential_subject_island":0,"blimp_superlative_quantifiers_1":0,"blimp_superlative_quantifiers_2":0,"blimp_tough_vs_raising_1":0,"blimp_tough_vs_raising_2":0,"blimp_transitive":0,"blimp_wh_island":0,"blimp_wh_questions_object_gap":0,"blimp_wh_questions_subject_gap":0,"blimp_wh_questions_subject_gap_long_distance":0,"blimp_wh_vs_that_no_gap":0,"blimp_wh_vs_that_no_gap_long_distance":0,"blimp_wh_vs_that_with_gap":0,"blimp_wh_vs_that_with_gap_long_distance":0,"commonsense_qa_text":0,"hellaswag":0,"piqa":0},"suite":"slm-benchmark-v1.1","task_config":{"arc_challenge":{"dataset_name":"ARC-Challenge","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_challenge.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_challenge","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"arc_easy":{"dataset_name":"ARC-Easy","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_easy.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_easy","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"blimp_adjunct_island":{"dataset_name":"adjunct_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/adjunct_island.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_adjunct_island","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_gender_agreement":{"dataset_name":"anaphor_gender_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_gender_agreement","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_number_agreement":{"dataset_name":"anaphor_number_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_number_agreement.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_number_agreement","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_passive":{"dataset_name":"animate_subject_passive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_passive.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_passive","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_trans":{"dataset_name":"animate_subject_trans","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_trans.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_trans","unsafe_code":false,"validation_split":"train"},"blimp_causative":{"dataset_name":"causative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/causative.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_causative","unsafe_code":false,"validation_split":"train"},"blimp_complex_NP_island":{"dataset_name":"complex_NP_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/complex_NP_island.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_complex_NP_island","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_complex_left_branch":{"dataset_name":"coordinate_structure_constraint_complex_left_branch","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_complex_left_branch","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_object_extraction":{"dataset_name":"coordinate_structure_constraint_object_extraction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_object_extraction","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_1":{"dataset_name":"determiner_noun_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_2":{"dataset_name":"determiner_noun_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_1":{"dataset_name":"determiner_noun_agreement_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_2":{"dataset_name":"determiner_noun_agreement_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_2":{"dataset_name":"determiner_noun_agreement_with_adj_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_1":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_2":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adjective_1":{"dataset_name":"determiner_noun_agreement_with_adjective_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adjective_1","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relational_noun":{"dataset_name":"distractor_agreement_relational_noun","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relational_noun","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relative_clause":{"dataset_name":"distractor_agreement_relative_clause","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relative_clause","unsafe_code":false,"validation_split":"train"},"blimp_drop_argument":{"dataset_name":"drop_argument","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/drop_argument.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_drop_argument","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_1":{"dataset_name":"ellipsis_n_bar_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_1","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_2":{"dataset_name":"ellipsis_n_bar_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_object_raising":{"dataset_name":"existential_there_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_object_raising.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_1":{"dataset_name":"existential_there_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_2":{"dataset_name":"existential_there_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_subject_raising":{"dataset_name":"existential_there_subject_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_subject_raising.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_subject_raising","unsafe_code":false,"validation_split":"train"},"blimp_expletive_it_object_raising":{"dataset_name":"expletive_it_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/expletive_it_object_raising.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_expletive_it_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_inchoative":{"dataset_name":"inchoative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/inchoative.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_inchoative","unsafe_code":false,"validation_split":"train"},"blimp_intransitive":{"dataset_name":"intransitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/intransitive.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_intransitive","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_adjectives":{"dataset_name":"irregular_past_participle_adjectives","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_adjectives","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_verbs":{"dataset_name":"irregular_past_participle_verbs","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_verbs","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_1":{"dataset_name":"irregular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_2":{"dataset_name":"irregular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_echo_question":{"dataset_name":"left_branch_island_echo_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_echo_question","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_simple_question":{"dataset_name":"left_branch_island_simple_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_simple_question","unsafe_code":false,"validation_split":"train"},"blimp_matrix_question_npi_licensor_present":{"dataset_name":"matrix_question_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_matrix_question_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_1":{"dataset_name":"npi_present_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_1.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_1","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_2":{"dataset_name":"npi_present_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_2.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_2","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_licensor_present":{"dataset_name":"only_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_scope":{"dataset_name":"only_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_passive_1":{"dataset_name":"passive_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_1.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_1","unsafe_code":false,"validation_split":"train"},"blimp_passive_2":{"dataset_name":"passive_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_2.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_c_command":{"dataset_name":"principle_A_c_command","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_c_command.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_c_command","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_1":{"dataset_name":"principle_A_case_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_1.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_2":{"dataset_name":"principle_A_case_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_2.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_1":{"dataset_name":"principle_A_domain_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_1.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_2":{"dataset_name":"principle_A_domain_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_2.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_3":{"dataset_name":"principle_A_domain_3","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_3.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_3","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_reconstruction":{"dataset_name":"principle_A_reconstruction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_reconstruction.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_reconstruction","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_1":{"dataset_name":"regular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_2":{"dataset_name":"regular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_licensor_present":{"dataset_name":"sentential_negation_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_scope":{"dataset_name":"sentential_negation_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_sentential_subject_island":{"dataset_name":"sentential_subject_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_subject_island.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_subject_island","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_1":{"dataset_name":"superlative_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_2":{"dataset_name":"superlative_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_1":{"dataset_name":"tough_vs_raising_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_1.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_1","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_2":{"dataset_name":"tough_vs_raising_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_2.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_2","unsafe_code":false,"validation_split":"train"},"blimp_transitive":{"dataset_name":"transitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/transitive.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_transitive","unsafe_code":false,"validation_split":"train"},"blimp_wh_island":{"dataset_name":"wh_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_island.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_island","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_object_gap":{"dataset_name":"wh_questions_object_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_object_gap.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_object_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap":{"dataset_name":"wh_questions_subject_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap_long_distance":{"dataset_name":"wh_questions_subject_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap":{"dataset_name":"wh_vs_that_no_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap_long_distance":{"dataset_name":"wh_vs_that_no_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap":{"dataset_name":"wh_vs_that_with_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap_long_distance":{"dataset_name":"wh_vs_that_with_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap_long_distance","unsafe_code":false,"validation_split":"train"},"commonsense_qa_text":{"dataset_path":"tau/commonsense_qa","description":"","doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/config/tasks/commonsense_qa_text.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"commonsense_qa_text","training_split":"train","unsafe_code":false,"validation_split":"validation"},"hellaswag":{"dataset_path":"Rowan/hellaswag","description":"","doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_config":{"doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":"<function process_docs at 0x7737c7db9260>","sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","process_docs":"def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\\n    def _process_doc(doc):\\n        ctx = doc[\\"ctx_a\\"] + \\" \\" + doc[\\"ctx_b\\"].capitalize()\\n        out_doc = {\\n            \\"query\\": preprocess(doc[\\"activity_label\\"] + \\": \\" + ctx),\\n            \\"choices\\": [preprocess(ending) for ending in doc[\\"endings\\"]],\\n            \\"gold\\": int(doc[\\"label\\"]),\\n        }\\n        return out_doc\\n\\n    return dataset.map(_process_doc)\\n","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"hellaswag","training_split":"train","unsafe_code":false,"validation_split":"validation"},"piqa":{"dataset_path":"baber/piqa","description":"","doc_to_choice":"{{[sol1, sol2]}}","doc_to_decontamination_query":"goal","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{[sol1, sol2]}}","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/piqa/piqa.yaml","dtype":"bfloat16","pretrained":"models/gpt2-medium-355m","tokenizer":"models/gpt2-medium-355m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"piqa","training_split":"train","unsafe_code":false,"validation_split":"validation"}},"task_versions":{"arc_challenge":1,"arc_easy":1,"blimp":"2.0","blimp_adjunct_island":1,"blimp_anaphor_gender_agreement":1,"blimp_anaphor_number_agreement":1,"blimp_animate_subject_passive":1,"blimp_animate_subject_trans":1,"blimp_causative":1,"blimp_complex_NP_island":1,"blimp_coordinate_structure_constraint_complex_left_branch":1,"blimp_coordinate_structure_constraint_object_extraction":1,"blimp_determiner_noun_agreement_1":1,"blimp_determiner_noun_agreement_2":1,"blimp_determiner_noun_agreement_irregular_1":1,"blimp_determiner_noun_agreement_irregular_2":1,"blimp_determiner_noun_agreement_with_adj_2":1,"blimp_determiner_noun_agreement_with_adj_irregular_1":1,"blimp_determiner_noun_agreement_with_adj_irregular_2":1,"blimp_determiner_noun_agreement_with_adjective_1":1,"blimp_distractor_agreement_relational_noun":1,"blimp_distractor_agreement_relative_clause":1,"blimp_drop_argument":1,"blimp_ellipsis_n_bar_1":1,"blimp_ellipsis_n_bar_2":1,"blimp_existential_there_object_raising":1,"blimp_existential_there_quantifiers_1":1,"blimp_existential_there_quantifiers_2":1,"blimp_existential_there_subject_raising":1,"blimp_expletive_it_object_raising":1,"blimp_inchoative":1,"blimp_intransitive":1,"blimp_irregular_past_participle_adjectives":1,"blimp_irregular_past_participle_verbs":1,"blimp_irregular_plural_subject_verb_agreement_1":1,"blimp_irregular_plural_subject_verb_agreement_2":1,"blimp_left_branch_island_echo_question":1,"blimp_left_branch_island_simple_question":1,"blimp_matrix_question_npi_licensor_present":1,"blimp_npi_present_1":1,"blimp_npi_present_2":1,"blimp_only_npi_licensor_present":1,"blimp_only_npi_scope":1,"blimp_passive_1":1,"blimp_passive_2":1,"blimp_principle_A_c_command":1,"blimp_principle_A_case_1":1,"blimp_principle_A_case_2":1,"blimp_principle_A_domain_1":1,"blimp_principle_A_domain_2":1,"blimp_principle_A_domain_3":1,"blimp_principle_A_reconstruction":1,"blimp_regular_plural_subject_verb_agreement_1":1,"blimp_regular_plural_subject_verb_agreement_2":1,"blimp_sentential_negation_npi_licensor_present":1,"blimp_sentential_negation_npi_scope":1,"blimp_sentential_subject_island":1,"blimp_superlative_quantifiers_1":1,"blimp_superlative_quantifiers_2":1,"blimp_tough_vs_raising_1":1,"blimp_tough_vs_raising_2":1,"blimp_transitive":1,"blimp_wh_island":1,"blimp_wh_questions_object_gap":1,"blimp_wh_questions_subject_gap":1,"blimp_wh_questions_subject_gap_long_distance":1,"blimp_wh_vs_that_no_gap":1,"blimp_wh_vs_that_no_gap_long_distance":1,"blimp_wh_vs_that_with_gap":1,"blimp_wh_vs_that_with_gap_long_distance":1,"commonsense_qa_text":"Yaml","hellaswag":1,"piqa":1}},"rank_range":[7,7],"release_date":"2022-03-02","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt2-medium-355m/20260723T113123Z","source":"openai-community/gpt2-medium","track":"base","updated_at":"2026-07-23T11:38:57.225513+00:00","vocab_size":50257},{"artifacts":{"result_file":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/mobilellm-r1-140m-base/20260716T161918Z/models__mobilellm-r1-140m-base/results_2026-07-16T18-27-38.828116.json","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/mobilellm-r1-140m-base/20260716T161918Z"},"benchmarks":{"arc_challenge":{"accuracy":0.24146757679180889,"chance":0.25,"ci":[-4.550625711035269,2.2753128555176305],"classification":"indistinguishable from chance","label":"ARC-Challenge","n_samples":1172,"score":-1.1376564277588153},"arc_easy":{"accuracy":0.49915824915824913,"chance":0.25,"ci":[30.47138047138047,35.746352413019075],"classification":"reliably above chance","label":"ARC-Easy","n_samples":2376,"score":33.22109988776655},"blimp":{"accuracy":0.8078363425925926,"categories":{"Anaphor agreement":0.9735,"Argument structure":0.7977777777777777,"Binding":0.7627142857142858,"Control / raising":0.8108000000000001,"Determiner-noun agreement":0.9315,"Ellipsis":0.892,"Filler-gap":0.767,"Irregular forms":0.9495,"Island effects":0.6956249999999999,"NPI licensing":0.6222857142857142,"Quantifiers":0.6245,"Subject-verb agreement":0.8668333333333335},"category_count":12,"chance":0.5,"ci":[60.729346891534426,61.79221445105819],"classification":"reliably above chance","label":"BLiMP","n_samples":70000,"score":61.567268518518524},"commonsense_qa":{"accuracy":0.3546273546273546,"chance":0.2,"ci":[15.950040950040947,22.809172809172804],"classification":"reliably above chance","label":"CommonsenseQA","n_samples":1221,"score":19.328419328419326},"hellaswag":{"accuracy":0.3406691894045011,"chance":0.25,"ci":[10.893580296089754,13.297815840138082],"classification":"reliably above chance","label":"HellaSwag","n_samples":10042,"score":12.089225253933478},"piqa":{"accuracy":0.6327529923830251,"chance":0.5,"ci":[22.19804134929271,30.794341675734493],"classification":"reliably above chance","label":"PIQA","n_samples":1838,"score":26.550598476605014}},"complete":true,"composite":{"ci":[30.06941936355877,32.21641211384041],"classification":"reliably above chance","score":31.211043675010572},"id":"mobilellm-r1-140m-base","name":"MobileLLM-R1 140M Base","params_m":140,"protocol":{"bootstrap_replicates":2000,"bootstrap_seed":20260716,"harness":"lm-eval==0.4.12","harness_git_hash":null,"inference_config":{"batch_size":"auto","batch_sizes":[64],"bootstrap_iters":100000,"device":"cuda:0","fewshot_seed":1234,"gen_kwargs":{},"limit":null,"model":"hf","model_args":{"dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false},"model_dtype":"torch.bfloat16","model_num_parameters":140248512,"model_revision":"main","model_sha":"","numpy_seed":1234,"random_seed":1234,"torch_seed":1234,"use_cache":null},"n_shot":{"arc_challenge":0,"arc_easy":0,"blimp":0,"blimp_adjunct_island":0,"blimp_anaphor_gender_agreement":0,"blimp_anaphor_number_agreement":0,"blimp_animate_subject_passive":0,"blimp_animate_subject_trans":0,"blimp_causative":0,"blimp_complex_NP_island":0,"blimp_coordinate_structure_constraint_complex_left_branch":0,"blimp_coordinate_structure_constraint_object_extraction":0,"blimp_determiner_noun_agreement_1":0,"blimp_determiner_noun_agreement_2":0,"blimp_determiner_noun_agreement_irregular_1":0,"blimp_determiner_noun_agreement_irregular_2":0,"blimp_determiner_noun_agreement_with_adj_2":0,"blimp_determiner_noun_agreement_with_adj_irregular_1":0,"blimp_determiner_noun_agreement_with_adj_irregular_2":0,"blimp_determiner_noun_agreement_with_adjective_1":0,"blimp_distractor_agreement_relational_noun":0,"blimp_distractor_agreement_relative_clause":0,"blimp_drop_argument":0,"blimp_ellipsis_n_bar_1":0,"blimp_ellipsis_n_bar_2":0,"blimp_existential_there_object_raising":0,"blimp_existential_there_quantifiers_1":0,"blimp_existential_there_quantifiers_2":0,"blimp_existential_there_subject_raising":0,"blimp_expletive_it_object_raising":0,"blimp_inchoative":0,"blimp_intransitive":0,"blimp_irregular_past_participle_adjectives":0,"blimp_irregular_past_participle_verbs":0,"blimp_irregular_plural_subject_verb_agreement_1":0,"blimp_irregular_plural_subject_verb_agreement_2":0,"blimp_left_branch_island_echo_question":0,"blimp_left_branch_island_simple_question":0,"blimp_matrix_question_npi_licensor_present":0,"blimp_npi_present_1":0,"blimp_npi_present_2":0,"blimp_only_npi_licensor_present":0,"blimp_only_npi_scope":0,"blimp_passive_1":0,"blimp_passive_2":0,"blimp_principle_A_c_command":0,"blimp_principle_A_case_1":0,"blimp_principle_A_case_2":0,"blimp_principle_A_domain_1":0,"blimp_principle_A_domain_2":0,"blimp_principle_A_domain_3":0,"blimp_principle_A_reconstruction":0,"blimp_regular_plural_subject_verb_agreement_1":0,"blimp_regular_plural_subject_verb_agreement_2":0,"blimp_sentential_negation_npi_licensor_present":0,"blimp_sentential_negation_npi_scope":0,"blimp_sentential_subject_island":0,"blimp_superlative_quantifiers_1":0,"blimp_superlative_quantifiers_2":0,"blimp_tough_vs_raising_1":0,"blimp_tough_vs_raising_2":0,"blimp_transitive":0,"blimp_wh_island":0,"blimp_wh_questions_object_gap":0,"blimp_wh_questions_subject_gap":0,"blimp_wh_questions_subject_gap_long_distance":0,"blimp_wh_vs_that_no_gap":0,"blimp_wh_vs_that_no_gap_long_distance":0,"blimp_wh_vs_that_with_gap":0,"blimp_wh_vs_that_with_gap_long_distance":0,"commonsense_qa_text":0,"hellaswag":0,"piqa":0},"suite":"slm-benchmark-v1.1","task_config":{"arc_challenge":{"dataset_name":"ARC-Challenge","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_challenge.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_challenge","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"arc_easy":{"dataset_name":"ARC-Easy","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_easy.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_easy","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"blimp_adjunct_island":{"dataset_name":"adjunct_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/adjunct_island.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_adjunct_island","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_gender_agreement":{"dataset_name":"anaphor_gender_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_gender_agreement","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_number_agreement":{"dataset_name":"anaphor_number_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_number_agreement.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_number_agreement","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_passive":{"dataset_name":"animate_subject_passive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_passive.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_passive","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_trans":{"dataset_name":"animate_subject_trans","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_trans.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_trans","unsafe_code":false,"validation_split":"train"},"blimp_causative":{"dataset_name":"causative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/causative.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_causative","unsafe_code":false,"validation_split":"train"},"blimp_complex_NP_island":{"dataset_name":"complex_NP_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/complex_NP_island.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_complex_NP_island","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_complex_left_branch":{"dataset_name":"coordinate_structure_constraint_complex_left_branch","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_complex_left_branch","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_object_extraction":{"dataset_name":"coordinate_structure_constraint_object_extraction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_object_extraction","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_1":{"dataset_name":"determiner_noun_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_2":{"dataset_name":"determiner_noun_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_1":{"dataset_name":"determiner_noun_agreement_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_2":{"dataset_name":"determiner_noun_agreement_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_2":{"dataset_name":"determiner_noun_agreement_with_adj_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_1":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_2":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adjective_1":{"dataset_name":"determiner_noun_agreement_with_adjective_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adjective_1","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relational_noun":{"dataset_name":"distractor_agreement_relational_noun","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relational_noun","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relative_clause":{"dataset_name":"distractor_agreement_relative_clause","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relative_clause","unsafe_code":false,"validation_split":"train"},"blimp_drop_argument":{"dataset_name":"drop_argument","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/drop_argument.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_drop_argument","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_1":{"dataset_name":"ellipsis_n_bar_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_1","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_2":{"dataset_name":"ellipsis_n_bar_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_object_raising":{"dataset_name":"existential_there_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_object_raising.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_1":{"dataset_name":"existential_there_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_2":{"dataset_name":"existential_there_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_subject_raising":{"dataset_name":"existential_there_subject_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_subject_raising.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_subject_raising","unsafe_code":false,"validation_split":"train"},"blimp_expletive_it_object_raising":{"dataset_name":"expletive_it_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/expletive_it_object_raising.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_expletive_it_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_inchoative":{"dataset_name":"inchoative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/inchoative.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_inchoative","unsafe_code":false,"validation_split":"train"},"blimp_intransitive":{"dataset_name":"intransitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/intransitive.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_intransitive","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_adjectives":{"dataset_name":"irregular_past_participle_adjectives","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_adjectives","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_verbs":{"dataset_name":"irregular_past_participle_verbs","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_verbs","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_1":{"dataset_name":"irregular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_2":{"dataset_name":"irregular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_echo_question":{"dataset_name":"left_branch_island_echo_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_echo_question","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_simple_question":{"dataset_name":"left_branch_island_simple_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_simple_question","unsafe_code":false,"validation_split":"train"},"blimp_matrix_question_npi_licensor_present":{"dataset_name":"matrix_question_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_matrix_question_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_1":{"dataset_name":"npi_present_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_1.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_1","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_2":{"dataset_name":"npi_present_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_2.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_2","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_licensor_present":{"dataset_name":"only_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_scope":{"dataset_name":"only_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_passive_1":{"dataset_name":"passive_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_1.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_1","unsafe_code":false,"validation_split":"train"},"blimp_passive_2":{"dataset_name":"passive_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_2.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_c_command":{"dataset_name":"principle_A_c_command","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_c_command.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_c_command","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_1":{"dataset_name":"principle_A_case_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_1.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_2":{"dataset_name":"principle_A_case_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_2.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_1":{"dataset_name":"principle_A_domain_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_1.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_2":{"dataset_name":"principle_A_domain_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_2.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_3":{"dataset_name":"principle_A_domain_3","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_3.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_3","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_reconstruction":{"dataset_name":"principle_A_reconstruction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_reconstruction.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_reconstruction","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_1":{"dataset_name":"regular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_2":{"dataset_name":"regular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_licensor_present":{"dataset_name":"sentential_negation_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_scope":{"dataset_name":"sentential_negation_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_sentential_subject_island":{"dataset_name":"sentential_subject_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_subject_island.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_subject_island","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_1":{"dataset_name":"superlative_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_2":{"dataset_name":"superlative_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_1":{"dataset_name":"tough_vs_raising_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_1.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_1","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_2":{"dataset_name":"tough_vs_raising_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_2.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_2","unsafe_code":false,"validation_split":"train"},"blimp_transitive":{"dataset_name":"transitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/transitive.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_transitive","unsafe_code":false,"validation_split":"train"},"blimp_wh_island":{"dataset_name":"wh_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_island.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_island","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_object_gap":{"dataset_name":"wh_questions_object_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_object_gap.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_object_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap":{"dataset_name":"wh_questions_subject_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap_long_distance":{"dataset_name":"wh_questions_subject_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap":{"dataset_name":"wh_vs_that_no_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap_long_distance":{"dataset_name":"wh_vs_that_no_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap":{"dataset_name":"wh_vs_that_with_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap_long_distance":{"dataset_name":"wh_vs_that_with_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap_long_distance","unsafe_code":false,"validation_split":"train"},"commonsense_qa_text":{"dataset_path":"tau/commonsense_qa","description":"","doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/config/tasks/commonsense_qa_text.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"commonsense_qa_text","training_split":"train","unsafe_code":false,"validation_split":"validation"},"hellaswag":{"dataset_path":"Rowan/hellaswag","description":"","doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_config":{"doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":"<function process_docs at 0x7166fab5e020>","sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","process_docs":"def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\\n    def _process_doc(doc):\\n        ctx = doc[\\"ctx_a\\"] + \\" \\" + doc[\\"ctx_b\\"].capitalize()\\n        out_doc = {\\n            \\"query\\": preprocess(doc[\\"activity_label\\"] + \\": \\" + ctx),\\n            \\"choices\\": [preprocess(ending) for ending in doc[\\"endings\\"]],\\n            \\"gold\\": int(doc[\\"label\\"]),\\n        }\\n        return out_doc\\n\\n    return dataset.map(_process_doc)\\n","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"hellaswag","training_split":"train","unsafe_code":false,"validation_split":"validation"},"piqa":{"dataset_path":"baber/piqa","description":"","doc_to_choice":"{{[sol1, sol2]}}","doc_to_decontamination_query":"goal","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{[sol1, sol2]}}","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/piqa/piqa.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"piqa","training_split":"train","unsafe_code":false,"validation_split":"validation"}},"task_versions":{"arc_challenge":1,"arc_easy":1,"blimp":"2.0","blimp_adjunct_island":1,"blimp_anaphor_gender_agreement":1,"blimp_anaphor_number_agreement":1,"blimp_animate_subject_passive":1,"blimp_animate_subject_trans":1,"blimp_causative":1,"blimp_complex_NP_island":1,"blimp_coordinate_structure_constraint_complex_left_branch":1,"blimp_coordinate_structure_constraint_object_extraction":1,"blimp_determiner_noun_agreement_1":1,"blimp_determiner_noun_agreement_2":1,"blimp_determiner_noun_agreement_irregular_1":1,"blimp_determiner_noun_agreement_irregular_2":1,"blimp_determiner_noun_agreement_with_adj_2":1,"blimp_determiner_noun_agreement_with_adj_irregular_1":1,"blimp_determiner_noun_agreement_with_adj_irregular_2":1,"blimp_determiner_noun_agreement_with_adjective_1":1,"blimp_distractor_agreement_relational_noun":1,"blimp_distractor_agreement_relative_clause":1,"blimp_drop_argument":1,"blimp_ellipsis_n_bar_1":1,"blimp_ellipsis_n_bar_2":1,"blimp_existential_there_object_raising":1,"blimp_existential_there_quantifiers_1":1,"blimp_existential_there_quantifiers_2":1,"blimp_existential_there_subject_raising":1,"blimp_expletive_it_object_raising":1,"blimp_inchoative":1,"blimp_intransitive":1,"blimp_irregular_past_participle_adjectives":1,"blimp_irregular_past_participle_verbs":1,"blimp_irregular_plural_subject_verb_agreement_1":1,"blimp_irregular_plural_subject_verb_agreement_2":1,"blimp_left_branch_island_echo_question":1,"blimp_left_branch_island_simple_question":1,"blimp_matrix_question_npi_licensor_present":1,"blimp_npi_present_1":1,"blimp_npi_present_2":1,"blimp_only_npi_licensor_present":1,"blimp_only_npi_scope":1,"blimp_passive_1":1,"blimp_passive_2":1,"blimp_principle_A_c_command":1,"blimp_principle_A_case_1":1,"blimp_principle_A_case_2":1,"blimp_principle_A_domain_1":1,"blimp_principle_A_domain_2":1,"blimp_principle_A_domain_3":1,"blimp_principle_A_reconstruction":1,"blimp_regular_plural_subject_verb_agreement_1":1,"blimp_regular_plural_subject_verb_agreement_2":1,"blimp_sentential_negation_npi_licensor_present":1,"blimp_sentential_negation_npi_scope":1,"blimp_sentential_subject_island":1,"blimp_superlative_quantifiers_1":1,"blimp_superlative_quantifiers_2":1,"blimp_tough_vs_raising_1":1,"blimp_tough_vs_raising_2":1,"blimp_transitive":1,"blimp_wh_island":1,"blimp_wh_questions_object_gap":1,"blimp_wh_questions_subject_gap":1,"blimp_wh_questions_subject_gap_long_distance":1,"blimp_wh_vs_that_no_gap":1,"blimp_wh_vs_that_no_gap_long_distance":1,"blimp_wh_vs_that_with_gap":1,"blimp_wh_vs_that_with_gap_long_distance":1,"commonsense_qa_text":"Yaml","hellaswag":1,"piqa":1}},"rank_range":[8,8],"release_date":"2025-09-10","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/mobilellm-r1-140m-base/20260716T161918Z","source":"facebook/MobileLLM-R1-140M-base","track":"base","updated_at":"2026-07-16T16:27:44.727364+00:00","vocab_size":128256},{"artifacts":{"result_file":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/models__baguettotron/results_2026-07-20T01-21-17.781285.json","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z"},"benchmarks":{"arc_challenge":{"accuracy":0.3037542662116041,"chance":0.25,"ci":[3.7542662116040924,10.58304891922638],"classification":"reliably above chance","label":"ARC-Challenge","n_samples":1172,"score":7.16723549488055},"arc_easy":{"accuracy":0.5058922558922558,"chance":0.25,"ci":[31.48148148148148,36.700336700336706],"classification":"reliably above chance","label":"ARC-Easy","n_samples":2376,"score":34.11896745230078},"blimp":{"accuracy":0.7847246362433862,"categories":{"Anaphor agreement":0.8355,"Argument structure":0.7777777777777778,"Binding":0.7462857142857143,"Control / raising":0.7584,"Determiner-noun agreement":0.941,"Ellipsis":0.88,"Filler-gap":0.7588571428571429,"Irregular forms":0.915,"Island effects":0.630375,"NPI licensing":0.623,"Quantifiers":0.7064999999999999,"Subject-verb agreement":0.8439999999999999},"category_count":12,"chance":0.5,"ci":[56.01578158068786,57.203216600529124],"classification":"reliably above chance","label":"BLiMP","n_samples":70000,"score":56.94492724867723},"commonsense_qa":{"accuracy":0.3063063063063063,"chance":0.2,"ci":[10.012285012285009,16.56429156429156],"classification":"reliably above chance","label":"CommonsenseQA","n_samples":1221,"score":13.288288288288284},"hellaswag":{"accuracy":0.35391356303525195,"chance":0.25,"ci":[12.607050388368853,15.050122817499838],"classification":"reliably above chance","label":"HellaSwag","n_samples":10042,"score":13.855141738033595},"piqa":{"accuracy":0.6207834602829162,"chance":0.5,"ci":[19.583786724700754,28.726877040261158],"classification":"reliably above chance","label":"PIQA","n_samples":1838,"score":24.15669205658324}},"complete":true,"composite":{"ci":[28.428847552063168,30.68468269023897],"classification":"reliably above chance","score":29.63151077505659},"id":"baguettotron","name":"Baguettotron","params_m":321,"protocol":{"bootstrap_replicates":2000,"bootstrap_seed":20260716,"harness":"lm-eval==0.4.12","harness_git_hash":"dc3c1cf","inference_config":{"batch_size":"auto","batch_sizes":[64],"bootstrap_iters":100000,"device":"cuda:0","fewshot_seed":1234,"gen_kwargs":{},"limit":null,"model":"hf","model_args":{"dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true},"model_dtype":"torch.bfloat16","model_num_parameters":320956992,"model_revision":"main","model_sha":"","numpy_seed":1234,"random_seed":1234,"torch_seed":1234,"use_cache":null},"n_shot":{"arc_challenge":0,"arc_easy":0,"blimp":0,"blimp_adjunct_island":0,"blimp_anaphor_gender_agreement":0,"blimp_anaphor_number_agreement":0,"blimp_animate_subject_passive":0,"blimp_animate_subject_trans":0,"blimp_causative":0,"blimp_complex_NP_island":0,"blimp_coordinate_structure_constraint_complex_left_branch":0,"blimp_coordinate_structure_constraint_object_extraction":0,"blimp_determiner_noun_agreement_1":0,"blimp_determiner_noun_agreement_2":0,"blimp_determiner_noun_agreement_irregular_1":0,"blimp_determiner_noun_agreement_irregular_2":0,"blimp_determiner_noun_agreement_with_adj_2":0,"blimp_determiner_noun_agreement_with_adj_irregular_1":0,"blimp_determiner_noun_agreement_with_adj_irregular_2":0,"blimp_determiner_noun_agreement_with_adjective_1":0,"blimp_distractor_agreement_relational_noun":0,"blimp_distractor_agreement_relative_clause":0,"blimp_drop_argument":0,"blimp_ellipsis_n_bar_1":0,"blimp_ellipsis_n_bar_2":0,"blimp_existential_there_object_raising":0,"blimp_existential_there_quantifiers_1":0,"blimp_existential_there_quantifiers_2":0,"blimp_existential_there_subject_raising":0,"blimp_expletive_it_object_raising":0,"blimp_inchoative":0,"blimp_intransitive":0,"blimp_irregular_past_participle_adjectives":0,"blimp_irregular_past_participle_verbs":0,"blimp_irregular_plural_subject_verb_agreement_1":0,"blimp_irregular_plural_subject_verb_agreement_2":0,"blimp_left_branch_island_echo_question":0,"blimp_left_branch_island_simple_question":0,"blimp_matrix_question_npi_licensor_present":0,"blimp_npi_present_1":0,"blimp_npi_present_2":0,"blimp_only_npi_licensor_present":0,"blimp_only_npi_scope":0,"blimp_passive_1":0,"blimp_passive_2":0,"blimp_principle_A_c_command":0,"blimp_principle_A_case_1":0,"blimp_principle_A_case_2":0,"blimp_principle_A_domain_1":0,"blimp_principle_A_domain_2":0,"blimp_principle_A_domain_3":0,"blimp_principle_A_reconstruction":0,"blimp_regular_plural_subject_verb_agreement_1":0,"blimp_regular_plural_subject_verb_agreement_2":0,"blimp_sentential_negation_npi_licensor_present":0,"blimp_sentential_negation_npi_scope":0,"blimp_sentential_subject_island":0,"blimp_superlative_quantifiers_1":0,"blimp_superlative_quantifiers_2":0,"blimp_tough_vs_raising_1":0,"blimp_tough_vs_raising_2":0,"blimp_transitive":0,"blimp_wh_island":0,"blimp_wh_questions_object_gap":0,"blimp_wh_questions_subject_gap":0,"blimp_wh_questions_subject_gap_long_distance":0,"blimp_wh_vs_that_no_gap":0,"blimp_wh_vs_that_no_gap_long_distance":0,"blimp_wh_vs_that_with_gap":0,"blimp_wh_vs_that_with_gap_long_distance":0,"commonsense_qa_text":0,"hellaswag":0,"piqa":0},"suite":"slm-benchmark-v1.1","task_config":{"arc_challenge":{"dataset_name":"ARC-Challenge","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_challenge.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_challenge","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"arc_easy":{"dataset_name":"ARC-Easy","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_easy.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_easy","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"blimp_adjunct_island":{"dataset_name":"adjunct_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/adjunct_island.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_adjunct_island","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_gender_agreement":{"dataset_name":"anaphor_gender_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_gender_agreement","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_number_agreement":{"dataset_name":"anaphor_number_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_number_agreement.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_number_agreement","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_passive":{"dataset_name":"animate_subject_passive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_passive.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_passive","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_trans":{"dataset_name":"animate_subject_trans","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_trans.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_trans","unsafe_code":false,"validation_split":"train"},"blimp_causative":{"dataset_name":"causative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/causative.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_causative","unsafe_code":false,"validation_split":"train"},"blimp_complex_NP_island":{"dataset_name":"complex_NP_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/complex_NP_island.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_complex_NP_island","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_complex_left_branch":{"dataset_name":"coordinate_structure_constraint_complex_left_branch","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_complex_left_branch","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_object_extraction":{"dataset_name":"coordinate_structure_constraint_object_extraction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_object_extraction","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_1":{"dataset_name":"determiner_noun_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_2":{"dataset_name":"determiner_noun_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_1":{"dataset_name":"determiner_noun_agreement_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_2":{"dataset_name":"determiner_noun_agreement_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_2":{"dataset_name":"determiner_noun_agreement_with_adj_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_1":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_2":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adjective_1":{"dataset_name":"determiner_noun_agreement_with_adjective_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adjective_1","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relational_noun":{"dataset_name":"distractor_agreement_relational_noun","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relational_noun","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relative_clause":{"dataset_name":"distractor_agreement_relative_clause","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relative_clause","unsafe_code":false,"validation_split":"train"},"blimp_drop_argument":{"dataset_name":"drop_argument","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/drop_argument.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_drop_argument","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_1":{"dataset_name":"ellipsis_n_bar_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_1","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_2":{"dataset_name":"ellipsis_n_bar_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_object_raising":{"dataset_name":"existential_there_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_object_raising.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_1":{"dataset_name":"existential_there_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_2":{"dataset_name":"existential_there_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_subject_raising":{"dataset_name":"existential_there_subject_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_subject_raising.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_subject_raising","unsafe_code":false,"validation_split":"train"},"blimp_expletive_it_object_raising":{"dataset_name":"expletive_it_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/expletive_it_object_raising.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_expletive_it_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_inchoative":{"dataset_name":"inchoative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/inchoative.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_inchoative","unsafe_code":false,"validation_split":"train"},"blimp_intransitive":{"dataset_name":"intransitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/intransitive.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_intransitive","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_adjectives":{"dataset_name":"irregular_past_participle_adjectives","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_adjectives","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_verbs":{"dataset_name":"irregular_past_participle_verbs","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_verbs","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_1":{"dataset_name":"irregular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_2":{"dataset_name":"irregular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_echo_question":{"dataset_name":"left_branch_island_echo_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_echo_question","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_simple_question":{"dataset_name":"left_branch_island_simple_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_simple_question","unsafe_code":false,"validation_split":"train"},"blimp_matrix_question_npi_licensor_present":{"dataset_name":"matrix_question_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_matrix_question_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_1":{"dataset_name":"npi_present_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_1.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_1","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_2":{"dataset_name":"npi_present_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_2.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_2","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_licensor_present":{"dataset_name":"only_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_scope":{"dataset_name":"only_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_passive_1":{"dataset_name":"passive_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_1.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_1","unsafe_code":false,"validation_split":"train"},"blimp_passive_2":{"dataset_name":"passive_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_2.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_c_command":{"dataset_name":"principle_A_c_command","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_c_command.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_c_command","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_1":{"dataset_name":"principle_A_case_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_1.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_2":{"dataset_name":"principle_A_case_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_2.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_1":{"dataset_name":"principle_A_domain_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_1.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_2":{"dataset_name":"principle_A_domain_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_2.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_3":{"dataset_name":"principle_A_domain_3","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_3.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_3","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_reconstruction":{"dataset_name":"principle_A_reconstruction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_reconstruction.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_reconstruction","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_1":{"dataset_name":"regular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_2":{"dataset_name":"regular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_licensor_present":{"dataset_name":"sentential_negation_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_scope":{"dataset_name":"sentential_negation_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_sentential_subject_island":{"dataset_name":"sentential_subject_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_subject_island.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_subject_island","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_1":{"dataset_name":"superlative_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_2":{"dataset_name":"superlative_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_1":{"dataset_name":"tough_vs_raising_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_1.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_1","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_2":{"dataset_name":"tough_vs_raising_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_2.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_2","unsafe_code":false,"validation_split":"train"},"blimp_transitive":{"dataset_name":"transitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/transitive.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_transitive","unsafe_code":false,"validation_split":"train"},"blimp_wh_island":{"dataset_name":"wh_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_island.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_island","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_object_gap":{"dataset_name":"wh_questions_object_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_object_gap.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_object_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap":{"dataset_name":"wh_questions_subject_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap_long_distance":{"dataset_name":"wh_questions_subject_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap":{"dataset_name":"wh_vs_that_no_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap_long_distance":{"dataset_name":"wh_vs_that_no_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap":{"dataset_name":"wh_vs_that_with_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap_long_distance":{"dataset_name":"wh_vs_that_with_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap_long_distance","unsafe_code":false,"validation_split":"train"},"commonsense_qa_text":{"dataset_path":"tau/commonsense_qa","description":"","doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/config/tasks/commonsense_qa_text.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"commonsense_qa_text","training_split":"train","unsafe_code":false,"validation_split":"validation"},"hellaswag":{"dataset_path":"Rowan/hellaswag","description":"","doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_config":{"doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":"<function process_docs at 0x7ed43ed7b560>","sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","process_docs":"def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\\n    def _process_doc(doc):\\n        ctx = doc[\\"ctx_a\\"] + \\" \\" + doc[\\"ctx_b\\"].capitalize()\\n        out_doc = {\\n            \\"query\\": preprocess(doc[\\"activity_label\\"] + \\": \\" + ctx),\\n            \\"choices\\": [preprocess(ending) for ending in doc[\\"endings\\"]],\\n            \\"gold\\": int(doc[\\"label\\"]),\\n        }\\n        return out_doc\\n\\n    return dataset.map(_process_doc)\\n","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"hellaswag","training_split":"train","unsafe_code":false,"validation_split":"validation"},"piqa":{"dataset_path":"baber/piqa","description":"","doc_to_choice":"{{[sol1, sol2]}}","doc_to_decontamination_query":"goal","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{[sol1, sol2]}}","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/piqa/piqa.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"piqa","training_split":"train","unsafe_code":false,"validation_split":"validation"}},"task_versions":{"arc_challenge":1,"arc_easy":1,"blimp":"2.0","blimp_adjunct_island":1,"blimp_anaphor_gender_agreement":1,"blimp_anaphor_number_agreement":1,"blimp_animate_subject_passive":1,"blimp_animate_subject_trans":1,"blimp_causative":1,"blimp_complex_NP_island":1,"blimp_coordinate_structure_constraint_complex_left_branch":1,"blimp_coordinate_structure_constraint_object_extraction":1,"blimp_determiner_noun_agreement_1":1,"blimp_determiner_noun_agreement_2":1,"blimp_determiner_noun_agreement_irregular_1":1,"blimp_determiner_noun_agreement_irregular_2":1,"blimp_determiner_noun_agreement_with_adj_2":1,"blimp_determiner_noun_agreement_with_adj_irregular_1":1,"blimp_determiner_noun_agreement_with_adj_irregular_2":1,"blimp_determiner_noun_agreement_with_adjective_1":1,"blimp_distractor_agreement_relational_noun":1,"blimp_distractor_agreement_relative_clause":1,"blimp_drop_argument":1,"blimp_ellipsis_n_bar_1":1,"blimp_ellipsis_n_bar_2":1,"blimp_existential_there_object_raising":1,"blimp_existential_there_quantifiers_1":1,"blimp_existential_there_quantifiers_2":1,"blimp_existential_there_subject_raising":1,"blimp_expletive_it_object_raising":1,"blimp_inchoative":1,"blimp_intransitive":1,"blimp_irregular_past_participle_adjectives":1,"blimp_irregular_past_participle_verbs":1,"blimp_irregular_plural_subject_verb_agreement_1":1,"blimp_irregular_plural_subject_verb_agreement_2":1,"blimp_left_branch_island_echo_question":1,"blimp_left_branch_island_simple_question":1,"blimp_matrix_question_npi_licensor_present":1,"blimp_npi_present_1":1,"blimp_npi_present_2":1,"blimp_only_npi_licensor_present":1,"blimp_only_npi_scope":1,"blimp_passive_1":1,"blimp_passive_2":1,"blimp_principle_A_c_command":1,"blimp_principle_A_case_1":1,"blimp_principle_A_case_2":1,"blimp_principle_A_domain_1":1,"blimp_principle_A_domain_2":1,"blimp_principle_A_domain_3":1,"blimp_principle_A_reconstruction":1,"blimp_regular_plural_subject_verb_agreement_1":1,"blimp_regular_plural_subject_verb_agreement_2":1,"blimp_sentential_negation_npi_licensor_present":1,"blimp_sentential_negation_npi_scope":1,"blimp_sentential_subject_island":1,"blimp_superlative_quantifiers_1":1,"blimp_superlative_quantifiers_2":1,"blimp_tough_vs_raising_1":1,"blimp_tough_vs_raising_2":1,"blimp_transitive":1,"blimp_wh_island":1,"blimp_wh_questions_object_gap":1,"blimp_wh_questions_subject_gap":1,"blimp_wh_questions_subject_gap_long_distance":1,"blimp_wh_vs_that_no_gap":1,"blimp_wh_vs_that_no_gap_long_distance":1,"blimp_wh_vs_that_with_gap":1,"blimp_wh_vs_that_with_gap_long_distance":1,"commonsense_qa_text":"Yaml","hellaswag":1,"piqa":1}},"rank_range":[9,9],"release_date":"2025-11-10","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z","source":"PleIAs/Baguettotron","track":"instruction","updated_at":"2026-07-19T23:21:23.658293+00:00","vocab_size":65536},{"artifacts":{"result_file":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt2-124m/20260716T154647Z-v1.1/results_v1.1_merged.json","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt2-124m/20260716T154647Z-v1.1"},"benchmarks":{"arc_challenge":{"accuracy":0.22440273037542663,"chance":0.25,"ci":[-6.598407281001136,-0.22753128555176305],"classification":"below chance","label":"ARC-Challenge","n_samples":1172,"score":-3.4129692832764498},"arc_easy":{"accuracy":0.3947811447811448,"chance":0.25,"ci":[16.665263748597084,21.885521885521886],"classification":"reliably above chance","label":"ARC-Easy","n_samples":2376,"score":19.304152637485974},"blimp":{"accuracy":0.8377077380952381,"categories":{"Anaphor agreement":0.997,"Argument structure":0.8356666666666667,"Binding":0.79,"Control / raising":0.8006,"Determiner-noun agreement":0.953875,"Ellipsis":0.848,"Filler-gap":0.8117142857142856,"Irregular forms":0.958,"Island effects":0.693625,"NPI licensing":0.7674285714285715,"Quantifiers":0.7177500000000001,"Subject-verb agreement":0.8788333333333332},"category_count":12,"chance":0.5,"ci":[66.76148528439154,67.78422933201058],"classification":"reliably above chance","label":"BLiMP","n_samples":70000,"score":67.54154761904762},"commonsense_qa":{"accuracy":0.29811629811629814,"chance":0.2,"ci":[9.193284193284189,15.54054054054054],"classification":"reliably above chance","label":"CommonsenseQA","n_samples":1221,"score":12.264537264537264},"hellaswag":{"accuracy":0.31139215295757816,"chance":0.25,"ci":[7.003916882427141,9.380601473809998],"classification":"reliably above chance","label":"HellaSwag","n_samples":10042,"score":8.185620394343756},"piqa":{"accuracy":0.6251360174102285,"chance":0.5,"ci":[20.67464635473342,29.37976060935801],"classification":"reliably above chance","label":"PIQA","n_samples":1838,"score":25.0272034820457}},"complete":true,"composite":{"ci":[26.76217046436712,28.856641117490366],"classification":"reliably above chance","score":27.871165134240197},"id":"gpt2-124m","name":"GPT-2 124M","params_m":124,"protocol":{"bootstrap_replicates":2000,"bootstrap_seed":20260716,"harness":"lm-eval==0.4.12","harness_git_hash":null,"inference_config":{"batch_size":"auto","batch_sizes":[64],"bootstrap_iters":100000,"device":"cuda:0","fewshot_seed":1234,"gen_kwargs":{},"limit":null,"model":"hf","model_args":{"dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false},"model_dtype":"torch.float32","model_num_parameters":124439808,"model_revision":"main","model_sha":"","numpy_seed":1234,"random_seed":1234,"tasks":["hellaswag","piqa","arc_easy","arc_challenge","commonsense_qa_text","blimp"],"torch_seed":1234,"use_cache":null,"v1_1_sources":{"rechecked_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt2-124m/20260716T154341Z-v1.1/results_v1.1_merged.json","unchanged_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt2-124m/20260716T154341Z-v1.1/results_v1.1_merged.json"}},"n_shot":{"arc_easy":0,"blimp":0,"blimp_adjunct_island":0,"blimp_anaphor_gender_agreement":0,"blimp_anaphor_number_agreement":0,"blimp_animate_subject_passive":0,"blimp_animate_subject_trans":0,"blimp_causative":0,"blimp_complex_NP_island":0,"blimp_coordinate_structure_constraint_complex_left_branch":0,"blimp_coordinate_structure_constraint_object_extraction":0,"blimp_determiner_noun_agreement_1":0,"blimp_determiner_noun_agreement_2":0,"blimp_determiner_noun_agreement_irregular_1":0,"blimp_determiner_noun_agreement_irregular_2":0,"blimp_determiner_noun_agreement_with_adj_2":0,"blimp_determiner_noun_agreement_with_adj_irregular_1":0,"blimp_determiner_noun_agreement_with_adj_irregular_2":0,"blimp_determiner_noun_agreement_with_adjective_1":0,"blimp_distractor_agreement_relational_noun":0,"blimp_distractor_agreement_relative_clause":0,"blimp_drop_argument":0,"blimp_ellipsis_n_bar_1":0,"blimp_ellipsis_n_bar_2":0,"blimp_existential_there_object_raising":0,"blimp_existential_there_quantifiers_1":0,"blimp_existential_there_quantifiers_2":0,"blimp_existential_there_subject_raising":0,"blimp_expletive_it_object_raising":0,"blimp_inchoative":0,"blimp_intransitive":0,"blimp_irregular_past_participle_adjectives":0,"blimp_irregular_past_participle_verbs":0,"blimp_irregular_plural_subject_verb_agreement_1":0,"blimp_irregular_plural_subject_verb_agreement_2":0,"blimp_left_branch_island_echo_question":0,"blimp_left_branch_island_simple_question":0,"blimp_matrix_question_npi_licensor_present":0,"blimp_npi_present_1":0,"blimp_npi_present_2":0,"blimp_only_npi_licensor_present":0,"blimp_only_npi_scope":0,"blimp_passive_1":0,"blimp_passive_2":0,"blimp_principle_A_c_command":0,"blimp_principle_A_case_1":0,"blimp_principle_A_case_2":0,"blimp_principle_A_domain_1":0,"blimp_principle_A_domain_2":0,"blimp_principle_A_domain_3":0,"blimp_principle_A_reconstruction":0,"blimp_regular_plural_subject_verb_agreement_1":0,"blimp_regular_plural_subject_verb_agreement_2":0,"blimp_sentential_negation_npi_licensor_present":0,"blimp_sentential_negation_npi_scope":0,"blimp_sentential_subject_island":0,"blimp_superlative_quantifiers_1":0,"blimp_superlative_quantifiers_2":0,"blimp_tough_vs_raising_1":0,"blimp_tough_vs_raising_2":0,"blimp_transitive":0,"blimp_wh_island":0,"blimp_wh_questions_object_gap":0,"blimp_wh_questions_subject_gap":0,"blimp_wh_questions_subject_gap_long_distance":0,"blimp_wh_vs_that_no_gap":0,"blimp_wh_vs_that_no_gap_long_distance":0,"blimp_wh_vs_that_with_gap":0,"blimp_wh_vs_that_with_gap_long_distance":0,"commonsense_qa":0,"hellaswag":0,"piqa":0},"suite":"slm-benchmark-v1.1","task_config":{"arc_challenge":{"dataset_name":"ARC-Challenge","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_challenge.yaml","dtype":"bfloat16","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_challenge","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"arc_easy":{"dataset_name":"ARC-Easy","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_easy.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_easy","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"blimp_adjunct_island":{"dataset_name":"adjunct_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/adjunct_island.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_adjunct_island","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_gender_agreement":{"dataset_name":"anaphor_gender_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_gender_agreement","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_number_agreement":{"dataset_name":"anaphor_number_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_number_agreement.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_number_agreement","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_passive":{"dataset_name":"animate_subject_passive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_passive.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_passive","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_trans":{"dataset_name":"animate_subject_trans","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_trans.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_trans","unsafe_code":false,"validation_split":"train"},"blimp_causative":{"dataset_name":"causative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/causative.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_causative","unsafe_code":false,"validation_split":"train"},"blimp_complex_NP_island":{"dataset_name":"complex_NP_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/complex_NP_island.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_complex_NP_island","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_complex_left_branch":{"dataset_name":"coordinate_structure_constraint_complex_left_branch","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_complex_left_branch","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_object_extraction":{"dataset_name":"coordinate_structure_constraint_object_extraction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_object_extraction","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_1":{"dataset_name":"determiner_noun_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_2":{"dataset_name":"determiner_noun_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_1":{"dataset_name":"determiner_noun_agreement_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_2":{"dataset_name":"determiner_noun_agreement_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_2":{"dataset_name":"determiner_noun_agreement_with_adj_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_1":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_2":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adjective_1":{"dataset_name":"determiner_noun_agreement_with_adjective_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adjective_1","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relational_noun":{"dataset_name":"distractor_agreement_relational_noun","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relational_noun","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relative_clause":{"dataset_name":"distractor_agreement_relative_clause","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relative_clause","unsafe_code":false,"validation_split":"train"},"blimp_drop_argument":{"dataset_name":"drop_argument","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/drop_argument.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_drop_argument","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_1":{"dataset_name":"ellipsis_n_bar_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_1","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_2":{"dataset_name":"ellipsis_n_bar_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_object_raising":{"dataset_name":"existential_there_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_object_raising.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_1":{"dataset_name":"existential_there_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_2":{"dataset_name":"existential_there_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_subject_raising":{"dataset_name":"existential_there_subject_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_subject_raising.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_subject_raising","unsafe_code":false,"validation_split":"train"},"blimp_expletive_it_object_raising":{"dataset_name":"expletive_it_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/expletive_it_object_raising.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_expletive_it_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_inchoative":{"dataset_name":"inchoative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/inchoative.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_inchoative","unsafe_code":false,"validation_split":"train"},"blimp_intransitive":{"dataset_name":"intransitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/intransitive.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_intransitive","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_adjectives":{"dataset_name":"irregular_past_participle_adjectives","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_adjectives","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_verbs":{"dataset_name":"irregular_past_participle_verbs","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_verbs","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_1":{"dataset_name":"irregular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_2":{"dataset_name":"irregular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_echo_question":{"dataset_name":"left_branch_island_echo_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_echo_question","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_simple_question":{"dataset_name":"left_branch_island_simple_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_simple_question","unsafe_code":false,"validation_split":"train"},"blimp_matrix_question_npi_licensor_present":{"dataset_name":"matrix_question_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_matrix_question_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_1":{"dataset_name":"npi_present_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_1.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_1","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_2":{"dataset_name":"npi_present_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_2.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_2","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_licensor_present":{"dataset_name":"only_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_licensor_present.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_scope":{"dataset_name":"only_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_scope.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_passive_1":{"dataset_name":"passive_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_1.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_1","unsafe_code":false,"validation_split":"train"},"blimp_passive_2":{"dataset_name":"passive_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_2.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_c_command":{"dataset_name":"principle_A_c_command","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_c_command.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_c_command","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_1":{"dataset_name":"principle_A_case_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_1.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_2":{"dataset_name":"principle_A_case_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_2.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_1":{"dataset_name":"principle_A_domain_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_1.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_2":{"dataset_name":"principle_A_domain_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_2.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_3":{"dataset_name":"principle_A_domain_3","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_3.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_3","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_reconstruction":{"dataset_name":"principle_A_reconstruction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_reconstruction.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_reconstruction","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_1":{"dataset_name":"regular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_2":{"dataset_name":"regular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_licensor_present":{"dataset_name":"sentential_negation_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_scope":{"dataset_name":"sentential_negation_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_sentential_subject_island":{"dataset_name":"sentential_subject_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_subject_island.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_subject_island","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_1":{"dataset_name":"superlative_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_2":{"dataset_name":"superlative_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_1":{"dataset_name":"tough_vs_raising_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_1.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_1","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_2":{"dataset_name":"tough_vs_raising_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_2.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_2","unsafe_code":false,"validation_split":"train"},"blimp_transitive":{"dataset_name":"transitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/transitive.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_transitive","unsafe_code":false,"validation_split":"train"},"blimp_wh_island":{"dataset_name":"wh_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_island.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_island","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_object_gap":{"dataset_name":"wh_questions_object_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_object_gap.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_object_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap":{"dataset_name":"wh_questions_subject_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap_long_distance":{"dataset_name":"wh_questions_subject_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap":{"dataset_name":"wh_vs_that_no_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap_long_distance":{"dataset_name":"wh_vs_that_no_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap":{"dataset_name":"wh_vs_that_with_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap_long_distance":{"dataset_name":"wh_vs_that_with_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap_long_distance","unsafe_code":false,"validation_split":"train"},"commonsense_qa_text":{"dataset_path":"tau/commonsense_qa","description":"","doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/config/tasks/commonsense_qa_text.yaml","dtype":"bfloat16","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"commonsense_qa_text","training_split":"train","unsafe_code":false,"validation_split":"validation"},"hellaswag":{"dataset_path":"Rowan/hellaswag","description":"","doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_config":{"doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":"<function process_docs at 0x707d2cd280e0>","sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","process_docs":"def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\\n    def _process_doc(doc):\\n        ctx = doc[\\"ctx_a\\"] + \\" \\" + doc[\\"ctx_b\\"].capitalize()\\n        out_doc = {\\n            \\"query\\": preprocess(doc[\\"activity_label\\"] + \\": \\" + ctx),\\n            \\"choices\\": [preprocess(ending) for ending in doc[\\"endings\\"]],\\n            \\"gold\\": int(doc[\\"label\\"]),\\n        }\\n        return out_doc\\n\\n    return dataset.map(_process_doc)\\n","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"hellaswag","training_split":"train","unsafe_code":false,"validation_split":"validation"},"piqa":{"dataset_path":"baber/piqa","description":"","doc_to_choice":"{{[sol1, sol2]}}","doc_to_decontamination_query":"goal","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{[sol1, sol2]}}","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/piqa/piqa.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"piqa","training_split":"train","unsafe_code":false,"validation_split":"validation"}},"task_versions":{"arc_challenge":1,"arc_easy":1,"blimp":"2.0","blimp_adjunct_island":1,"blimp_anaphor_gender_agreement":1,"blimp_anaphor_number_agreement":1,"blimp_animate_subject_passive":1,"blimp_animate_subject_trans":1,"blimp_causative":1,"blimp_complex_NP_island":1,"blimp_coordinate_structure_constraint_complex_left_branch":1,"blimp_coordinate_structure_constraint_object_extraction":1,"blimp_determiner_noun_agreement_1":1,"blimp_determiner_noun_agreement_2":1,"blimp_determiner_noun_agreement_irregular_1":1,"blimp_determiner_noun_agreement_irregular_2":1,"blimp_determiner_noun_agreement_with_adj_2":1,"blimp_determiner_noun_agreement_with_adj_irregular_1":1,"blimp_determiner_noun_agreement_with_adj_irregular_2":1,"blimp_determiner_noun_agreement_with_adjective_1":1,"blimp_distractor_agreement_relational_noun":1,"blimp_distractor_agreement_relative_clause":1,"blimp_drop_argument":1,"blimp_ellipsis_n_bar_1":1,"blimp_ellipsis_n_bar_2":1,"blimp_existential_there_object_raising":1,"blimp_existential_there_quantifiers_1":1,"blimp_existential_there_quantifiers_2":1,"blimp_existential_there_subject_raising":1,"blimp_expletive_it_object_raising":1,"blimp_inchoative":1,"blimp_intransitive":1,"blimp_irregular_past_participle_adjectives":1,"blimp_irregular_past_participle_verbs":1,"blimp_irregular_plural_subject_verb_agreement_1":1,"blimp_irregular_plural_subject_verb_agreement_2":1,"blimp_left_branch_island_echo_question":1,"blimp_left_branch_island_simple_question":1,"blimp_matrix_question_npi_licensor_present":1,"blimp_npi_present_1":1,"blimp_npi_present_2":1,"blimp_only_npi_licensor_present":1,"blimp_only_npi_scope":1,"blimp_passive_1":1,"blimp_passive_2":1,"blimp_principle_A_c_command":1,"blimp_principle_A_case_1":1,"blimp_principle_A_case_2":1,"blimp_principle_A_domain_1":1,"blimp_principle_A_domain_2":1,"blimp_principle_A_domain_3":1,"blimp_principle_A_reconstruction":1,"blimp_regular_plural_subject_verb_agreement_1":1,"blimp_regular_plural_subject_verb_agreement_2":1,"blimp_sentential_negation_npi_licensor_present":1,"blimp_sentential_negation_npi_scope":1,"blimp_sentential_subject_island":1,"blimp_superlative_quantifiers_1":1,"blimp_superlative_quantifiers_2":1,"blimp_tough_vs_raising_1":1,"blimp_tough_vs_raising_2":1,"blimp_transitive":1,"blimp_wh_island":1,"blimp_wh_questions_object_gap":1,"blimp_wh_questions_subject_gap":1,"blimp_wh_questions_subject_gap_long_distance":1,"blimp_wh_vs_that_no_gap":1,"blimp_wh_vs_that_no_gap_long_distance":1,"blimp_wh_vs_that_with_gap":1,"blimp_wh_vs_that_with_gap_long_distance":1,"commonsense_qa_text":"Yaml","hellaswag":1,"piqa":1}},"rank_range":[10,11],"release_date":"2022-03-02","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt2-124m/20260716T154647Z-v1.1","source":"openai-community/gpt2","track":"base","updated_at":"2026-07-16T15:46:55.978943+00:00","vocab_size":50257},{"artifacts":{"result_file":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T154647Z-v1.1/results_v1.1_merged.json","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T154647Z-v1.1"},"benchmarks":{"arc_challenge":{"accuracy":0.24914675767918087,"chance":0.25,"ci":[-3.526734926052333,3.1854379977246903],"classification":"indistinguishable from chance","label":"ARC-Challenge","n_samples":1172,"score":-0.11376564277588337},"arc_easy":{"accuracy":0.4562289562289562,"chance":0.25,"ci":[24.74747474747475,30.1921997755331],"classification":"reliably above chance","label":"ARC-Easy","n_samples":2376,"score":27.497194163860826},"blimp":{"accuracy":0.7947869047619047,"categories":{"Anaphor agreement":0.913,"Argument structure":0.7863333333333333,"Binding":0.7452857142857142,"Control / raising":0.8018000000000001,"Determiner-noun agreement":0.9412499999999999,"Ellipsis":0.857,"Filler-gap":0.7838571428571429,"Irregular forms":0.8845000000000001,"Island effects":0.63425,"NPI licensing":0.624,"Quantifiers":0.6865,"Subject-verb agreement":0.8796666666666667},"category_count":12,"chance":0.5,"ci":[57.94834143518519,59.13693105158729],"classification":"reliably above chance","label":"BLiMP","n_samples":70000,"score":58.957380952380944},"commonsense_qa":{"accuracy":0.29975429975429974,"chance":0.2,"ci":[9.193284193284189,15.74529074529074],"classification":"reliably above chance","label":"CommonsenseQA","n_samples":1221,"score":12.469287469287464},"hellaswag":{"accuracy":0.31627165903206533,"chance":0.25,"ci":[7.6279625572595116,9.991369581092746],"classification":"reliably above chance","label":"HellaSwag","n_samples":10042,"score":8.836221204275377},"piqa":{"accuracy":0.6169749727965179,"chance":0.5,"ci":[18.933623503808494,27.967899891186054],"classification":"reliably above chance","label":"PIQA","n_samples":1838,"score":23.394994559303584}},"complete":true,"composite":{"ci":[26.15776710993005,28.30631083392145],"classification":"reliably above chance","score":27.338042663290054},"id":"supra-50m-base","name":"Supra 50M Base","params_m":51.78624,"protocol":{"bootstrap_replicates":2000,"bootstrap_seed":20260716,"harness":"lm-eval==0.4.12","harness_git_hash":null,"inference_config":{"batch_size":"auto","batch_sizes":[64],"bootstrap_iters":100000,"device":"cuda:0","fewshot_seed":1234,"gen_kwargs":{},"limit":null,"model":"hf","model_args":{"dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true},"model_dtype":"torch.bfloat16","model_num_parameters":51786240,"model_revision":"main","model_sha":"","numpy_seed":1234,"random_seed":1234,"tasks":["hellaswag","piqa","arc_easy","arc_challenge","commonsense_qa_text","blimp"],"torch_seed":1234,"use_cache":null,"v1_1_sources":{"rechecked_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T154341Z-v1.1/results_v1.1_merged.json","unchanged_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T154341Z-v1.1/results_v1.1_merged.json"}},"n_shot":{"arc_easy":0,"blimp":0,"blimp_adjunct_island":0,"blimp_anaphor_gender_agreement":0,"blimp_anaphor_number_agreement":0,"blimp_animate_subject_passive":0,"blimp_animate_subject_trans":0,"blimp_causative":0,"blimp_complex_NP_island":0,"blimp_coordinate_structure_constraint_complex_left_branch":0,"blimp_coordinate_structure_constraint_object_extraction":0,"blimp_determiner_noun_agreement_1":0,"blimp_determiner_noun_agreement_2":0,"blimp_determiner_noun_agreement_irregular_1":0,"blimp_determiner_noun_agreement_irregular_2":0,"blimp_determiner_noun_agreement_with_adj_2":0,"blimp_determiner_noun_agreement_with_adj_irregular_1":0,"blimp_determiner_noun_agreement_with_adj_irregular_2":0,"blimp_determiner_noun_agreement_with_adjective_1":0,"blimp_distractor_agreement_relational_noun":0,"blimp_distractor_agreement_relative_clause":0,"blimp_drop_argument":0,"blimp_ellipsis_n_bar_1":0,"blimp_ellipsis_n_bar_2":0,"blimp_existential_there_object_raising":0,"blimp_existential_there_quantifiers_1":0,"blimp_existential_there_quantifiers_2":0,"blimp_existential_there_subject_raising":0,"blimp_expletive_it_object_raising":0,"blimp_inchoative":0,"blimp_intransitive":0,"blimp_irregular_past_participle_adjectives":0,"blimp_irregular_past_participle_verbs":0,"blimp_irregular_plural_subject_verb_agreement_1":0,"blimp_irregular_plural_subject_verb_agreement_2":0,"blimp_left_branch_island_echo_question":0,"blimp_left_branch_island_simple_question":0,"blimp_matrix_question_npi_licensor_present":0,"blimp_npi_present_1":0,"blimp_npi_present_2":0,"blimp_only_npi_licensor_present":0,"blimp_only_npi_scope":0,"blimp_passive_1":0,"blimp_passive_2":0,"blimp_principle_A_c_command":0,"blimp_principle_A_case_1":0,"blimp_principle_A_case_2":0,"blimp_principle_A_domain_1":0,"blimp_principle_A_domain_2":0,"blimp_principle_A_domain_3":0,"blimp_principle_A_reconstruction":0,"blimp_regular_plural_subject_verb_agreement_1":0,"blimp_regular_plural_subject_verb_agreement_2":0,"blimp_sentential_negation_npi_licensor_present":0,"blimp_sentential_negation_npi_scope":0,"blimp_sentential_subject_island":0,"blimp_superlative_quantifiers_1":0,"blimp_superlative_quantifiers_2":0,"blimp_tough_vs_raising_1":0,"blimp_tough_vs_raising_2":0,"blimp_transitive":0,"blimp_wh_island":0,"blimp_wh_questions_object_gap":0,"blimp_wh_questions_subject_gap":0,"blimp_wh_questions_subject_gap_long_distance":0,"blimp_wh_vs_that_no_gap":0,"blimp_wh_vs_that_no_gap_long_distance":0,"blimp_wh_vs_that_with_gap":0,"blimp_wh_vs_that_with_gap_long_distance":0,"commonsense_qa":0,"hellaswag":0,"piqa":0},"suite":"slm-benchmark-v1.1","task_config":{"arc_challenge":{"dataset_name":"ARC-Challenge","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_challenge.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T153917Z/tokenizer_compat","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_challenge","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"arc_easy":{"dataset_name":"ARC-Easy","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_easy.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_easy","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"blimp_adjunct_island":{"dataset_name":"adjunct_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/adjunct_island.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_adjunct_island","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_gender_agreement":{"dataset_name":"anaphor_gender_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_gender_agreement","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_number_agreement":{"dataset_name":"anaphor_number_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_number_agreement.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_number_agreement","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_passive":{"dataset_name":"animate_subject_passive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_passive.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_passive","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_trans":{"dataset_name":"animate_subject_trans","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_trans.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_trans","unsafe_code":false,"validation_split":"train"},"blimp_causative":{"dataset_name":"causative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/causative.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_causative","unsafe_code":false,"validation_split":"train"},"blimp_complex_NP_island":{"dataset_name":"complex_NP_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/complex_NP_island.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_complex_NP_island","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_complex_left_branch":{"dataset_name":"coordinate_structure_constraint_complex_left_branch","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_complex_left_branch","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_object_extraction":{"dataset_name":"coordinate_structure_constraint_object_extraction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_object_extraction","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_1":{"dataset_name":"determiner_noun_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_2":{"dataset_name":"determiner_noun_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_1":{"dataset_name":"determiner_noun_agreement_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_2":{"dataset_name":"determiner_noun_agreement_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_2":{"dataset_name":"determiner_noun_agreement_with_adj_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_1":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_2":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adjective_1":{"dataset_name":"determiner_noun_agreement_with_adjective_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adjective_1","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relational_noun":{"dataset_name":"distractor_agreement_relational_noun","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relational_noun","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relative_clause":{"dataset_name":"distractor_agreement_relative_clause","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relative_clause","unsafe_code":false,"validation_split":"train"},"blimp_drop_argument":{"dataset_name":"drop_argument","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/drop_argument.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_drop_argument","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_1":{"dataset_name":"ellipsis_n_bar_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_1","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_2":{"dataset_name":"ellipsis_n_bar_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_object_raising":{"dataset_name":"existential_there_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_object_raising.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_1":{"dataset_name":"existential_there_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_2":{"dataset_name":"existential_there_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_subject_raising":{"dataset_name":"existential_there_subject_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_subject_raising.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_subject_raising","unsafe_code":false,"validation_split":"train"},"blimp_expletive_it_object_raising":{"dataset_name":"expletive_it_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/expletive_it_object_raising.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_expletive_it_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_inchoative":{"dataset_name":"inchoative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/inchoative.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_inchoative","unsafe_code":false,"validation_split":"train"},"blimp_intransitive":{"dataset_name":"intransitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/intransitive.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_intransitive","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_adjectives":{"dataset_name":"irregular_past_participle_adjectives","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_adjectives","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_verbs":{"dataset_name":"irregular_past_participle_verbs","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_verbs","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_1":{"dataset_name":"irregular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_2":{"dataset_name":"irregular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_echo_question":{"dataset_name":"left_branch_island_echo_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_echo_question","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_simple_question":{"dataset_name":"left_branch_island_simple_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_simple_question","unsafe_code":false,"validation_split":"train"},"blimp_matrix_question_npi_licensor_present":{"dataset_name":"matrix_question_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_matrix_question_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_1":{"dataset_name":"npi_present_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_1.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_1","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_2":{"dataset_name":"npi_present_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_2.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_2","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_licensor_present":{"dataset_name":"only_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_scope":{"dataset_name":"only_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_passive_1":{"dataset_name":"passive_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_1.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_1","unsafe_code":false,"validation_split":"train"},"blimp_passive_2":{"dataset_name":"passive_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_2.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_c_command":{"dataset_name":"principle_A_c_command","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_c_command.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_c_command","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_1":{"dataset_name":"principle_A_case_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_1.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_2":{"dataset_name":"principle_A_case_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_2.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_1":{"dataset_name":"principle_A_domain_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_1.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_2":{"dataset_name":"principle_A_domain_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_2.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_3":{"dataset_name":"principle_A_domain_3","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_3.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_3","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_reconstruction":{"dataset_name":"principle_A_reconstruction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_reconstruction.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_reconstruction","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_1":{"dataset_name":"regular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_2":{"dataset_name":"regular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_licensor_present":{"dataset_name":"sentential_negation_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_scope":{"dataset_name":"sentential_negation_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_sentential_subject_island":{"dataset_name":"sentential_subject_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_subject_island.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_subject_island","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_1":{"dataset_name":"superlative_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_2":{"dataset_name":"superlative_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_1":{"dataset_name":"tough_vs_raising_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_1.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_1","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_2":{"dataset_name":"tough_vs_raising_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_2.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_2","unsafe_code":false,"validation_split":"train"},"blimp_transitive":{"dataset_name":"transitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/transitive.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_transitive","unsafe_code":false,"validation_split":"train"},"blimp_wh_island":{"dataset_name":"wh_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_island.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_island","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_object_gap":{"dataset_name":"wh_questions_object_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_object_gap.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_object_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap":{"dataset_name":"wh_questions_subject_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap_long_distance":{"dataset_name":"wh_questions_subject_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap":{"dataset_name":"wh_vs_that_no_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap_long_distance":{"dataset_name":"wh_vs_that_no_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap":{"dataset_name":"wh_vs_that_with_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap_long_distance":{"dataset_name":"wh_vs_that_with_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap_long_distance","unsafe_code":false,"validation_split":"train"},"commonsense_qa_text":{"dataset_path":"tau/commonsense_qa","description":"","doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/config/tasks/commonsense_qa_text.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T153917Z/tokenizer_compat","trust_remote_code":false},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"commonsense_qa_text","training_split":"train","unsafe_code":false,"validation_split":"validation"},"hellaswag":{"dataset_path":"Rowan/hellaswag","description":"","doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_config":{"doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":"<function process_docs at 0x7e92921a09a0>","sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","process_docs":"def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\\n    def _process_doc(doc):\\n        ctx = doc[\\"ctx_a\\"] + \\" \\" + doc[\\"ctx_b\\"].capitalize()\\n        out_doc = {\\n            \\"query\\": preprocess(doc[\\"activity_label\\"] + \\": \\" + ctx),\\n            \\"choices\\": [preprocess(ending) for ending in doc[\\"endings\\"]],\\n            \\"gold\\": int(doc[\\"label\\"]),\\n        }\\n        return out_doc\\n\\n    return dataset.map(_process_doc)\\n","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"hellaswag","training_split":"train","unsafe_code":false,"validation_split":"validation"},"piqa":{"dataset_path":"baber/piqa","description":"","doc_to_choice":"{{[sol1, sol2]}}","doc_to_decontamination_query":"goal","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{[sol1, sol2]}}","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/piqa/piqa.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"piqa","training_split":"train","unsafe_code":false,"validation_split":"validation"}},"task_versions":{"arc_challenge":1,"arc_easy":1,"blimp":"2.0","blimp_adjunct_island":1,"blimp_anaphor_gender_agreement":1,"blimp_anaphor_number_agreement":1,"blimp_animate_subject_passive":1,"blimp_animate_subject_trans":1,"blimp_causative":1,"blimp_complex_NP_island":1,"blimp_coordinate_structure_constraint_complex_left_branch":1,"blimp_coordinate_structure_constraint_object_extraction":1,"blimp_determiner_noun_agreement_1":1,"blimp_determiner_noun_agreement_2":1,"blimp_determiner_noun_agreement_irregular_1":1,"blimp_determiner_noun_agreement_irregular_2":1,"blimp_determiner_noun_agreement_with_adj_2":1,"blimp_determiner_noun_agreement_with_adj_irregular_1":1,"blimp_determiner_noun_agreement_with_adj_irregular_2":1,"blimp_determiner_noun_agreement_with_adjective_1":1,"blimp_distractor_agreement_relational_noun":1,"blimp_distractor_agreement_relative_clause":1,"blimp_drop_argument":1,"blimp_ellipsis_n_bar_1":1,"blimp_ellipsis_n_bar_2":1,"blimp_existential_there_object_raising":1,"blimp_existential_there_quantifiers_1":1,"blimp_existential_there_quantifiers_2":1,"blimp_existential_there_subject_raising":1,"blimp_expletive_it_object_raising":1,"blimp_inchoative":1,"blimp_intransitive":1,"blimp_irregular_past_participle_adjectives":1,"blimp_irregular_past_participle_verbs":1,"blimp_irregular_plural_subject_verb_agreement_1":1,"blimp_irregular_plural_subject_verb_agreement_2":1,"blimp_left_branch_island_echo_question":1,"blimp_left_branch_island_simple_question":1,"blimp_matrix_question_npi_licensor_present":1,"blimp_npi_present_1":1,"blimp_npi_present_2":1,"blimp_only_npi_licensor_present":1,"blimp_only_npi_scope":1,"blimp_passive_1":1,"blimp_passive_2":1,"blimp_principle_A_c_command":1,"blimp_principle_A_case_1":1,"blimp_principle_A_case_2":1,"blimp_principle_A_domain_1":1,"blimp_principle_A_domain_2":1,"blimp_principle_A_domain_3":1,"blimp_principle_A_reconstruction":1,"blimp_regular_plural_subject_verb_agreement_1":1,"blimp_regular_plural_subject_verb_agreement_2":1,"blimp_sentential_negation_npi_licensor_present":1,"blimp_sentential_negation_npi_scope":1,"blimp_sentential_subject_island":1,"blimp_superlative_quantifiers_1":1,"blimp_superlative_quantifiers_2":1,"blimp_tough_vs_raising_1":1,"blimp_tough_vs_raising_2":1,"blimp_transitive":1,"blimp_wh_island":1,"blimp_wh_questions_object_gap":1,"blimp_wh_questions_subject_gap":1,"blimp_wh_questions_subject_gap_long_distance":1,"blimp_wh_vs_that_no_gap":1,"blimp_wh_vs_that_no_gap_long_distance":1,"blimp_wh_vs_that_with_gap":1,"blimp_wh_vs_that_with_gap_long_distance":1,"commonsense_qa_text":"Yaml","hellaswag":1,"piqa":1}},"rank_range":[10,11],"release_date":"2026-05-21","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T154647Z-v1.1","source":"SupraLabs/Supra-50M-Base","track":"base","updated_at":"2026-07-16T15:47:07.077007+00:00","vocab_size":32000},{"artifacts":{"result_file":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/models__veyra2-apricot-50m-base/results_2026-07-16T18-37-33.567087.json","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z"},"benchmarks":{"arc_challenge":{"accuracy":0.23464163822525597,"chance":0.25,"ci":[-5.233219567690558,1.137656427758819],"classification":"indistinguishable from chance","label":"ARC-Challenge","n_samples":1172,"score":-2.047781569965871},"arc_easy":{"accuracy":0.4276094276094276,"chance":0.25,"ci":[20.93153759820427,26.430976430976433],"classification":"reliably above chance","label":"ARC-Easy","n_samples":2376,"score":23.681257014590347},"blimp":{"accuracy":0.7935405423280423,"categories":{"Anaphor agreement":0.9115,"Argument structure":0.7822222222222223,"Binding":0.7711428571428571,"Control / raising":0.7988,"Determiner-noun agreement":0.933125,"Ellipsis":0.849,"Filler-gap":0.7371428571428572,"Irregular forms":0.907,"Island effects":0.676375,"NPI licensing":0.6524285714285715,"Quantifiers":0.67225,"Subject-verb agreement":0.8315},"category_count":12,"chance":0.5,"ci":[57.73771858465608,58.93828703703703],"classification":"reliably above chance","label":"BLiMP","n_samples":70000,"score":58.708108465608454},"commonsense_qa":{"accuracy":0.28665028665028663,"chance":0.2,"ci":[7.657657657657656,14.004914004914006],"classification":"reliably above chance","label":"CommonsenseQA","n_samples":1221,"score":10.831285831285827},"hellaswag":{"accuracy":0.3127862975502888,"chance":0.25,"ci":[7.203080395671517,9.540264223594237],"classification":"reliably above chance","label":"HellaSwag","n_samples":10042,"score":8.371506340038508},"piqa":{"accuracy":0.6169749727965179,"chance":0.5,"ci":[19.0424374319913,27.856365614798694],"classification":"reliably above chance","label":"PIQA","n_samples":1838,"score":23.394994559303584}},"complete":true,"composite":{"ci":[24.919591817425154,27.053619924780357],"classification":"reliably above chance","score":26.088907580286257},"id":"veyra2-apricot-50m-base","name":"Veyra2 Apricot 50M Base","params_m":49.3,"protocol":{"bootstrap_replicates":2000,"bootstrap_seed":20260716,"harness":"lm-eval==0.4.12","harness_git_hash":null,"inference_config":{"batch_size":"64","batch_sizes":[],"bootstrap_iters":100000,"device":"cuda:0","fewshot_seed":1234,"gen_kwargs":{},"limit":null,"model":"hf","model_args":{"dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true},"model_dtype":"torch.bfloat16","model_num_parameters":49303040,"model_revision":"main","model_sha":"","numpy_seed":1234,"random_seed":1234,"torch_seed":1234,"use_cache":null},"n_shot":{"arc_challenge":0,"arc_easy":0,"blimp":0,"blimp_adjunct_island":0,"blimp_anaphor_gender_agreement":0,"blimp_anaphor_number_agreement":0,"blimp_animate_subject_passive":0,"blimp_animate_subject_trans":0,"blimp_causative":0,"blimp_complex_NP_island":0,"blimp_coordinate_structure_constraint_complex_left_branch":0,"blimp_coordinate_structure_constraint_object_extraction":0,"blimp_determiner_noun_agreement_1":0,"blimp_determiner_noun_agreement_2":0,"blimp_determiner_noun_agreement_irregular_1":0,"blimp_determiner_noun_agreement_irregular_2":0,"blimp_determiner_noun_agreement_with_adj_2":0,"blimp_determiner_noun_agreement_with_adj_irregular_1":0,"blimp_determiner_noun_agreement_with_adj_irregular_2":0,"blimp_determiner_noun_agreement_with_adjective_1":0,"blimp_distractor_agreement_relational_noun":0,"blimp_distractor_agreement_relative_clause":0,"blimp_drop_argument":0,"blimp_ellipsis_n_bar_1":0,"blimp_ellipsis_n_bar_2":0,"blimp_existential_there_object_raising":0,"blimp_existential_there_quantifiers_1":0,"blimp_existential_there_quantifiers_2":0,"blimp_existential_there_subject_raising":0,"blimp_expletive_it_object_raising":0,"blimp_inchoative":0,"blimp_intransitive":0,"blimp_irregular_past_participle_adjectives":0,"blimp_irregular_past_participle_verbs":0,"blimp_irregular_plural_subject_verb_agreement_1":0,"blimp_irregular_plural_subject_verb_agreement_2":0,"blimp_left_branch_island_echo_question":0,"blimp_left_branch_island_simple_question":0,"blimp_matrix_question_npi_licensor_present":0,"blimp_npi_present_1":0,"blimp_npi_present_2":0,"blimp_only_npi_licensor_present":0,"blimp_only_npi_scope":0,"blimp_passive_1":0,"blimp_passive_2":0,"blimp_principle_A_c_command":0,"blimp_principle_A_case_1":0,"blimp_principle_A_case_2":0,"blimp_principle_A_domain_1":0,"blimp_principle_A_domain_2":0,"blimp_principle_A_domain_3":0,"blimp_principle_A_reconstruction":0,"blimp_regular_plural_subject_verb_agreement_1":0,"blimp_regular_plural_subject_verb_agreement_2":0,"blimp_sentential_negation_npi_licensor_present":0,"blimp_sentential_negation_npi_scope":0,"blimp_sentential_subject_island":0,"blimp_superlative_quantifiers_1":0,"blimp_superlative_quantifiers_2":0,"blimp_tough_vs_raising_1":0,"blimp_tough_vs_raising_2":0,"blimp_transitive":0,"blimp_wh_island":0,"blimp_wh_questions_object_gap":0,"blimp_wh_questions_subject_gap":0,"blimp_wh_questions_subject_gap_long_distance":0,"blimp_wh_vs_that_no_gap":0,"blimp_wh_vs_that_no_gap_long_distance":0,"blimp_wh_vs_that_with_gap":0,"blimp_wh_vs_that_with_gap_long_distance":0,"commonsense_qa_text":0,"hellaswag":0,"piqa":0},"suite":"slm-benchmark-v1.1","task_config":{"arc_challenge":{"dataset_name":"ARC-Challenge","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_challenge.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_challenge","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"arc_easy":{"dataset_name":"ARC-Easy","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_easy.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_easy","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"blimp_adjunct_island":{"dataset_name":"adjunct_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/adjunct_island.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_adjunct_island","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_gender_agreement":{"dataset_name":"anaphor_gender_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_gender_agreement","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_number_agreement":{"dataset_name":"anaphor_number_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_number_agreement.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_number_agreement","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_passive":{"dataset_name":"animate_subject_passive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_passive.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_passive","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_trans":{"dataset_name":"animate_subject_trans","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_trans.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_trans","unsafe_code":false,"validation_split":"train"},"blimp_causative":{"dataset_name":"causative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/causative.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_causative","unsafe_code":false,"validation_split":"train"},"blimp_complex_NP_island":{"dataset_name":"complex_NP_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/complex_NP_island.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_complex_NP_island","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_complex_left_branch":{"dataset_name":"coordinate_structure_constraint_complex_left_branch","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_complex_left_branch","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_object_extraction":{"dataset_name":"coordinate_structure_constraint_object_extraction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_object_extraction","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_1":{"dataset_name":"determiner_noun_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_2":{"dataset_name":"determiner_noun_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_1":{"dataset_name":"determiner_noun_agreement_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_2":{"dataset_name":"determiner_noun_agreement_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_2":{"dataset_name":"determiner_noun_agreement_with_adj_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_1":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_2":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adjective_1":{"dataset_name":"determiner_noun_agreement_with_adjective_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adjective_1","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relational_noun":{"dataset_name":"distractor_agreement_relational_noun","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relational_noun","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relative_clause":{"dataset_name":"distractor_agreement_relative_clause","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relative_clause","unsafe_code":false,"validation_split":"train"},"blimp_drop_argument":{"dataset_name":"drop_argument","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/drop_argument.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_drop_argument","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_1":{"dataset_name":"ellipsis_n_bar_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_1","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_2":{"dataset_name":"ellipsis_n_bar_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_object_raising":{"dataset_name":"existential_there_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_object_raising.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_1":{"dataset_name":"existential_there_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_2":{"dataset_name":"existential_there_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_subject_raising":{"dataset_name":"existential_there_subject_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_subject_raising.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_subject_raising","unsafe_code":false,"validation_split":"train"},"blimp_expletive_it_object_raising":{"dataset_name":"expletive_it_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/expletive_it_object_raising.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_expletive_it_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_inchoative":{"dataset_name":"inchoative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/inchoative.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_inchoative","unsafe_code":false,"validation_split":"train"},"blimp_intransitive":{"dataset_name":"intransitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/intransitive.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_intransitive","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_adjectives":{"dataset_name":"irregular_past_participle_adjectives","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_adjectives","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_verbs":{"dataset_name":"irregular_past_participle_verbs","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_verbs","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_1":{"dataset_name":"irregular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_2":{"dataset_name":"irregular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_echo_question":{"dataset_name":"left_branch_island_echo_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_echo_question","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_simple_question":{"dataset_name":"left_branch_island_simple_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_simple_question","unsafe_code":false,"validation_split":"train"},"blimp_matrix_question_npi_licensor_present":{"dataset_name":"matrix_question_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_matrix_question_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_1":{"dataset_name":"npi_present_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_1.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_1","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_2":{"dataset_name":"npi_present_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_2.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_2","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_licensor_present":{"dataset_name":"only_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_scope":{"dataset_name":"only_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_passive_1":{"dataset_name":"passive_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_1.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_1","unsafe_code":false,"validation_split":"train"},"blimp_passive_2":{"dataset_name":"passive_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_2.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_c_command":{"dataset_name":"principle_A_c_command","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_c_command.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_c_command","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_1":{"dataset_name":"principle_A_case_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_1.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_2":{"dataset_name":"principle_A_case_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_2.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_1":{"dataset_name":"principle_A_domain_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_1.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_2":{"dataset_name":"principle_A_domain_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_2.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_3":{"dataset_name":"principle_A_domain_3","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_3.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_3","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_reconstruction":{"dataset_name":"principle_A_reconstruction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_reconstruction.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_reconstruction","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_1":{"dataset_name":"regular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_2":{"dataset_name":"regular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_licensor_present":{"dataset_name":"sentential_negation_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_scope":{"dataset_name":"sentential_negation_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_sentential_subject_island":{"dataset_name":"sentential_subject_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_subject_island.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_subject_island","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_1":{"dataset_name":"superlative_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_2":{"dataset_name":"superlative_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_1":{"dataset_name":"tough_vs_raising_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_1.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_1","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_2":{"dataset_name":"tough_vs_raising_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_2.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_2","unsafe_code":false,"validation_split":"train"},"blimp_transitive":{"dataset_name":"transitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/transitive.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_transitive","unsafe_code":false,"validation_split":"train"},"blimp_wh_island":{"dataset_name":"wh_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_island.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_island","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_object_gap":{"dataset_name":"wh_questions_object_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_object_gap.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_object_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap":{"dataset_name":"wh_questions_subject_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap_long_distance":{"dataset_name":"wh_questions_subject_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap":{"dataset_name":"wh_vs_that_no_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap_long_distance":{"dataset_name":"wh_vs_that_no_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap":{"dataset_name":"wh_vs_that_with_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap_long_distance":{"dataset_name":"wh_vs_that_with_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap_long_distance","unsafe_code":false,"validation_split":"train"},"commonsense_qa_text":{"dataset_path":"tau/commonsense_qa","description":"","doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/config/tasks/commonsense_qa_text.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"commonsense_qa_text","training_split":"train","unsafe_code":false,"validation_split":"validation"},"hellaswag":{"dataset_path":"Rowan/hellaswag","description":"","doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_config":{"doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":"<function process_docs at 0x7c9c447b31a0>","sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","process_docs":"def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\\n    def _process_doc(doc):\\n        ctx = doc[\\"ctx_a\\"] + \\" \\" + doc[\\"ctx_b\\"].capitalize()\\n        out_doc = {\\n            \\"query\\": preprocess(doc[\\"activity_label\\"] + \\": \\" + ctx),\\n            \\"choices\\": [preprocess(ending) for ending in doc[\\"endings\\"]],\\n            \\"gold\\": int(doc[\\"label\\"]),\\n        }\\n        return out_doc\\n\\n    return dataset.map(_process_doc)\\n","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"hellaswag","training_split":"train","unsafe_code":false,"validation_split":"validation"},"piqa":{"dataset_path":"baber/piqa","description":"","doc_to_choice":"{{[sol1, sol2]}}","doc_to_decontamination_query":"goal","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{[sol1, sol2]}}","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/piqa/piqa.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"piqa","training_split":"train","unsafe_code":false,"validation_split":"validation"}},"task_versions":{"arc_challenge":1,"arc_easy":1,"blimp":"2.0","blimp_adjunct_island":1,"blimp_anaphor_gender_agreement":1,"blimp_anaphor_number_agreement":1,"blimp_animate_subject_passive":1,"blimp_animate_subject_trans":1,"blimp_causative":1,"blimp_complex_NP_island":1,"blimp_coordinate_structure_constraint_complex_left_branch":1,"blimp_coordinate_structure_constraint_object_extraction":1,"blimp_determiner_noun_agreement_1":1,"blimp_determiner_noun_agreement_2":1,"blimp_determiner_noun_agreement_irregular_1":1,"blimp_determiner_noun_agreement_irregular_2":1,"blimp_determiner_noun_agreement_with_adj_2":1,"blimp_determiner_noun_agreement_with_adj_irregular_1":1,"blimp_determiner_noun_agreement_with_adj_irregular_2":1,"blimp_determiner_noun_agreement_with_adjective_1":1,"blimp_distractor_agreement_relational_noun":1,"blimp_distractor_agreement_relative_clause":1,"blimp_drop_argument":1,"blimp_ellipsis_n_bar_1":1,"blimp_ellipsis_n_bar_2":1,"blimp_existential_there_object_raising":1,"blimp_existential_there_quantifiers_1":1,"blimp_existential_there_quantifiers_2":1,"blimp_existential_there_subject_raising":1,"blimp_expletive_it_object_raising":1,"blimp_inchoative":1,"blimp_intransitive":1,"blimp_irregular_past_participle_adjectives":1,"blimp_irregular_past_participle_verbs":1,"blimp_irregular_plural_subject_verb_agreement_1":1,"blimp_irregular_plural_subject_verb_agreement_2":1,"blimp_left_branch_island_echo_question":1,"blimp_left_branch_island_simple_question":1,"blimp_matrix_question_npi_licensor_present":1,"blimp_npi_present_1":1,"blimp_npi_present_2":1,"blimp_only_npi_licensor_present":1,"blimp_only_npi_scope":1,"blimp_passive_1":1,"blimp_passive_2":1,"blimp_principle_A_c_command":1,"blimp_principle_A_case_1":1,"blimp_principle_A_case_2":1,"blimp_principle_A_domain_1":1,"blimp_principle_A_domain_2":1,"blimp_principle_A_domain_3":1,"blimp_principle_A_reconstruction":1,"blimp_regular_plural_subject_verb_agreement_1":1,"blimp_regular_plural_subject_verb_agreement_2":1,"blimp_sentential_negation_npi_licensor_present":1,"blimp_sentential_negation_npi_scope":1,"blimp_sentential_subject_island":1,"blimp_superlative_quantifiers_1":1,"blimp_superlative_quantifiers_2":1,"blimp_tough_vs_raising_1":1,"blimp_tough_vs_raising_2":1,"blimp_transitive":1,"blimp_wh_island":1,"blimp_wh_questions_object_gap":1,"blimp_wh_questions_subject_gap":1,"blimp_wh_questions_subject_gap_long_distance":1,"blimp_wh_vs_that_no_gap":1,"blimp_wh_vs_that_no_gap_long_distance":1,"blimp_wh_vs_that_with_gap":1,"blimp_wh_vs_that_with_gap_long_distance":1,"commonsense_qa_text":"Yaml","hellaswag":1,"piqa":1}},"rank_range":[12,12],"release_date":"2026-06-14","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z","source":"veyra-ai/Veyra2-Apricot-50M-Base","track":"base","updated_at":"2026-07-16T16:37:38.898399+00:00","vocab_size":8192},{"artifacts":{"result_file":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/falcon-h1-tiny-r-90m/20260720T072155Z/models__falcon-h1-tiny-r-90m/results_2026-07-20T09-28-46.371565.json","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/falcon-h1-tiny-r-90m/20260720T072155Z"},"benchmarks":{"arc_challenge":{"accuracy":0.24914675767918087,"chance":0.25,"ci":[-3.415813424345847,3.1854379977246903],"classification":"indistinguishable from chance","label":"ARC-Challenge","n_samples":1172,"score":-0.11376564277588337},"arc_easy":{"accuracy":0.3926767676767677,"chance":0.25,"ci":[16.273849607182942,21.49410774410773],"classification":"reliably above chance","label":"ARC-Easy","n_samples":2376,"score":19.023569023569024},"blimp":{"accuracy":0.7126083333333333,"categories":{"Anaphor agreement":0.8009999999999999,"Argument structure":0.6073333333333334,"Binding":0.7232857142857142,"Control / raising":0.6587999999999999,"Determiner-noun agreement":0.8799999999999999,"Ellipsis":0.871,"Filler-gap":0.7327142857142858,"Irregular forms":0.781,"Island effects":0.5905,"NPI licensing":0.704,"Quantifiers":0.44050000000000006,"Subject-verb agreement":0.7611666666666667},"category_count":12,"chance":0.5,"ci":[41.63552711640213,42.98391468253967],"classification":"reliably above chance","label":"BLiMP","n_samples":70000,"score":42.52166666666666},"commonsense_qa":{"accuracy":0.2833742833742834,"chance":0.2,"ci":[7.350532350532346,13.595413595413595],"classification":"reliably above chance","label":"CommonsenseQA","n_samples":1221,"score":10.42178542178542},"hellaswag":{"accuracy":0.30770762796255724,"chance":0.25,"ci":[6.552479585739893,8.876053906924254],"classification":"reliably above chance","label":"HellaSwag","n_samples":10042,"score":7.694350395007632},"piqa":{"accuracy":0.6077257889009793,"chance":0.5,"ci":[16.97497279651796,25.897714907508163],"classification":"reliably above chance","label":"PIQA","n_samples":1838,"score":21.545157780195858}},"complete":true,"composite":{"ci":[19.64362638928377,21.818589193772866],"classification":"reliably above chance","score":20.800441105410517},"id":"falcon-h1-tiny-r-90m","name":"Falcon H1 Tiny R 90M","params_m":90,"protocol":{"bootstrap_replicates":2000,"bootstrap_seed":20260716,"harness":"lm-eval==0.4.12","harness_git_hash":"dc3c1cf","inference_config":{"batch_size":"auto","batch_sizes":[64],"bootstrap_iters":100000,"device":"cuda:0","fewshot_seed":1234,"gen_kwargs":{},"limit":null,"model":"hf","model_args":{"dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false},"model_dtype":"torch.bfloat16","model_num_parameters":91131072,"model_revision":"main","model_sha":"","numpy_seed":1234,"random_seed":1234,"torch_seed":1234,"use_cache":null},"n_shot":{"arc_challenge":0,"arc_easy":0,"blimp":0,"blimp_adjunct_island":0,"blimp_anaphor_gender_agreement":0,"blimp_anaphor_number_agreement":0,"blimp_animate_subject_passive":0,"blimp_animate_subject_trans":0,"blimp_causative":0,"blimp_complex_NP_island":0,"blimp_coordinate_structure_constraint_complex_left_branch":0,"blimp_coordinate_structure_constraint_object_extraction":0,"blimp_determiner_noun_agreement_1":0,"blimp_determiner_noun_agreement_2":0,"blimp_determiner_noun_agreement_irregular_1":0,"blimp_determiner_noun_agreement_irregular_2":0,"blimp_determiner_noun_agreement_with_adj_2":0,"blimp_determiner_noun_agreement_with_adj_irregular_1":0,"blimp_determiner_noun_agreement_with_adj_irregular_2":0,"blimp_determiner_noun_agreement_with_adjective_1":0,"blimp_distractor_agreement_relational_noun":0,"blimp_distractor_agreement_relative_clause":0,"blimp_drop_argument":0,"blimp_ellipsis_n_bar_1":0,"blimp_ellipsis_n_bar_2":0,"blimp_existential_there_object_raising":0,"blimp_existential_there_quantifiers_1":0,"blimp_existential_there_quantifiers_2":0,"blimp_existential_there_subject_raising":0,"blimp_expletive_it_object_raising":0,"blimp_inchoative":0,"blimp_intransitive":0,"blimp_irregular_past_participle_adjectives":0,"blimp_irregular_past_participle_verbs":0,"blimp_irregular_plural_subject_verb_agreement_1":0,"blimp_irregular_plural_subject_verb_agreement_2":0,"blimp_left_branch_island_echo_question":0,"blimp_left_branch_island_simple_question":0,"blimp_matrix_question_npi_licensor_present":0,"blimp_npi_present_1":0,"blimp_npi_present_2":0,"blimp_only_npi_licensor_present":0,"blimp_only_npi_scope":0,"blimp_passive_1":0,"blimp_passive_2":0,"blimp_principle_A_c_command":0,"blimp_principle_A_case_1":0,"blimp_principle_A_case_2":0,"blimp_principle_A_domain_1":0,"blimp_principle_A_domain_2":0,"blimp_principle_A_domain_3":0,"blimp_principle_A_reconstruction":0,"blimp_regular_plural_subject_verb_agreement_1":0,"blimp_regular_plural_subject_verb_agreement_2":0,"blimp_sentential_negation_npi_licensor_present":0,"blimp_sentential_negation_npi_scope":0,"blimp_sentential_subject_island":0,"blimp_superlative_quantifiers_1":0,"blimp_superlative_quantifiers_2":0,"blimp_tough_vs_raising_1":0,"blimp_tough_vs_raising_2":0,"blimp_transitive":0,"blimp_wh_island":0,"blimp_wh_questions_object_gap":0,"blimp_wh_questions_subject_gap":0,"blimp_wh_questions_subject_gap_long_distance":0,"blimp_wh_vs_that_no_gap":0,"blimp_wh_vs_that_no_gap_long_distance":0,"blimp_wh_vs_that_with_gap":0,"blimp_wh_vs_that_with_gap_long_distance":0,"commonsense_qa_text":0,"hellaswag":0,"piqa":0},"suite":"slm-benchmark-v1.1","task_config":{"arc_challenge":{"dataset_name":"ARC-Challenge","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_challenge.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_challenge","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"arc_easy":{"dataset_name":"ARC-Easy","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_easy.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_easy","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"blimp_adjunct_island":{"dataset_name":"adjunct_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/adjunct_island.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_adjunct_island","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_gender_agreement":{"dataset_name":"anaphor_gender_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_gender_agreement","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_number_agreement":{"dataset_name":"anaphor_number_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_number_agreement.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_number_agreement","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_passive":{"dataset_name":"animate_subject_passive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_passive.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_passive","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_trans":{"dataset_name":"animate_subject_trans","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_trans.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_trans","unsafe_code":false,"validation_split":"train"},"blimp_causative":{"dataset_name":"causative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/causative.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_causative","unsafe_code":false,"validation_split":"train"},"blimp_complex_NP_island":{"dataset_name":"complex_NP_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/complex_NP_island.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_complex_NP_island","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_complex_left_branch":{"dataset_name":"coordinate_structure_constraint_complex_left_branch","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_complex_left_branch","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_object_extraction":{"dataset_name":"coordinate_structure_constraint_object_extraction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_object_extraction","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_1":{"dataset_name":"determiner_noun_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_2":{"dataset_name":"determiner_noun_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_1":{"dataset_name":"determiner_noun_agreement_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_2":{"dataset_name":"determiner_noun_agreement_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_2":{"dataset_name":"determiner_noun_agreement_with_adj_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_1":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_2":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adjective_1":{"dataset_name":"determiner_noun_agreement_with_adjective_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adjective_1","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relational_noun":{"dataset_name":"distractor_agreement_relational_noun","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relational_noun","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relative_clause":{"dataset_name":"distractor_agreement_relative_clause","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relative_clause","unsafe_code":false,"validation_split":"train"},"blimp_drop_argument":{"dataset_name":"drop_argument","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/drop_argument.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_drop_argument","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_1":{"dataset_name":"ellipsis_n_bar_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_1","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_2":{"dataset_name":"ellipsis_n_bar_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_object_raising":{"dataset_name":"existential_there_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_object_raising.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_1":{"dataset_name":"existential_there_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_2":{"dataset_name":"existential_there_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_subject_raising":{"dataset_name":"existential_there_subject_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_subject_raising.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_subject_raising","unsafe_code":false,"validation_split":"train"},"blimp_expletive_it_object_raising":{"dataset_name":"expletive_it_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/expletive_it_object_raising.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_expletive_it_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_inchoative":{"dataset_name":"inchoative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/inchoative.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_inchoative","unsafe_code":false,"validation_split":"train"},"blimp_intransitive":{"dataset_name":"intransitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/intransitive.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_intransitive","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_adjectives":{"dataset_name":"irregular_past_participle_adjectives","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_adjectives","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_verbs":{"dataset_name":"irregular_past_participle_verbs","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_verbs","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_1":{"dataset_name":"irregular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_2":{"dataset_name":"irregular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_echo_question":{"dataset_name":"left_branch_island_echo_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_echo_question","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_simple_question":{"dataset_name":"left_branch_island_simple_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_simple_question","unsafe_code":false,"validation_split":"train"},"blimp_matrix_question_npi_licensor_present":{"dataset_name":"matrix_question_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_matrix_question_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_1":{"dataset_name":"npi_present_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_1.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_1","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_2":{"dataset_name":"npi_present_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_2.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_2","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_licensor_present":{"dataset_name":"only_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_scope":{"dataset_name":"only_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_passive_1":{"dataset_name":"passive_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_1.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_1","unsafe_code":false,"validation_split":"train"},"blimp_passive_2":{"dataset_name":"passive_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_2.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_c_command":{"dataset_name":"principle_A_c_command","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_c_command.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_c_command","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_1":{"dataset_name":"principle_A_case_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_1.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_2":{"dataset_name":"principle_A_case_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_2.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_1":{"dataset_name":"principle_A_domain_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_1.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_2":{"dataset_name":"principle_A_domain_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_2.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_3":{"dataset_name":"principle_A_domain_3","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_3.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_3","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_reconstruction":{"dataset_name":"principle_A_reconstruction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_reconstruction.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_reconstruction","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_1":{"dataset_name":"regular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_2":{"dataset_name":"regular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_licensor_present":{"dataset_name":"sentential_negation_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_scope":{"dataset_name":"sentential_negation_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_sentential_subject_island":{"dataset_name":"sentential_subject_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_subject_island.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_subject_island","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_1":{"dataset_name":"superlative_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_2":{"dataset_name":"superlative_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_1":{"dataset_name":"tough_vs_raising_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_1.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_1","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_2":{"dataset_name":"tough_vs_raising_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_2.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_2","unsafe_code":false,"validation_split":"train"},"blimp_transitive":{"dataset_name":"transitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/transitive.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_transitive","unsafe_code":false,"validation_split":"train"},"blimp_wh_island":{"dataset_name":"wh_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_island.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_island","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_object_gap":{"dataset_name":"wh_questions_object_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_object_gap.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_object_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap":{"dataset_name":"wh_questions_subject_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap_long_distance":{"dataset_name":"wh_questions_subject_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap":{"dataset_name":"wh_vs_that_no_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap_long_distance":{"dataset_name":"wh_vs_that_no_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap":{"dataset_name":"wh_vs_that_with_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap_long_distance":{"dataset_name":"wh_vs_that_with_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap_long_distance","unsafe_code":false,"validation_split":"train"},"commonsense_qa_text":{"dataset_path":"tau/commonsense_qa","description":"","doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/config/tasks/commonsense_qa_text.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"commonsense_qa_text","training_split":"train","unsafe_code":false,"validation_split":"validation"},"hellaswag":{"dataset_path":"Rowan/hellaswag","description":"","doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_config":{"doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":"<function process_docs at 0x7cc583ab0cc0>","sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","process_docs":"def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\\n    def _process_doc(doc):\\n        ctx = doc[\\"ctx_a\\"] + \\" \\" + doc[\\"ctx_b\\"].capitalize()\\n        out_doc = {\\n            \\"query\\": preprocess(doc[\\"activity_label\\"] + \\": \\" + ctx),\\n            \\"choices\\": [preprocess(ending) for ending in doc[\\"endings\\"]],\\n            \\"gold\\": int(doc[\\"label\\"]),\\n        }\\n        return out_doc\\n\\n    return dataset.map(_process_doc)\\n","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"hellaswag","training_split":"train","unsafe_code":false,"validation_split":"validation"},"piqa":{"dataset_path":"baber/piqa","description":"","doc_to_choice":"{{[sol1, sol2]}}","doc_to_decontamination_query":"goal","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{[sol1, sol2]}}","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/piqa/piqa.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"piqa","training_split":"train","unsafe_code":false,"validation_split":"validation"}},"task_versions":{"arc_challenge":1,"arc_easy":1,"blimp":"2.0","blimp_adjunct_island":1,"blimp_anaphor_gender_agreement":1,"blimp_anaphor_number_agreement":1,"blimp_animate_subject_passive":1,"blimp_animate_subject_trans":1,"blimp_causative":1,"blimp_complex_NP_island":1,"blimp_coordinate_structure_constraint_complex_left_branch":1,"blimp_coordinate_structure_constraint_object_extraction":1,"blimp_determiner_noun_agreement_1":1,"blimp_determiner_noun_agreement_2":1,"blimp_determiner_noun_agreement_irregular_1":1,"blimp_determiner_noun_agreement_irregular_2":1,"blimp_determiner_noun_agreement_with_adj_2":1,"blimp_determiner_noun_agreement_with_adj_irregular_1":1,"blimp_determiner_noun_agreement_with_adj_irregular_2":1,"blimp_determiner_noun_agreement_with_adjective_1":1,"blimp_distractor_agreement_relational_noun":1,"blimp_distractor_agreement_relative_clause":1,"blimp_drop_argument":1,"blimp_ellipsis_n_bar_1":1,"blimp_ellipsis_n_bar_2":1,"blimp_existential_there_object_raising":1,"blimp_existential_there_quantifiers_1":1,"blimp_existential_there_quantifiers_2":1,"blimp_existential_there_subject_raising":1,"blimp_expletive_it_object_raising":1,"blimp_inchoative":1,"blimp_intransitive":1,"blimp_irregular_past_participle_adjectives":1,"blimp_irregular_past_participle_verbs":1,"blimp_irregular_plural_subject_verb_agreement_1":1,"blimp_irregular_plural_subject_verb_agreement_2":1,"blimp_left_branch_island_echo_question":1,"blimp_left_branch_island_simple_question":1,"blimp_matrix_question_npi_licensor_present":1,"blimp_npi_present_1":1,"blimp_npi_present_2":1,"blimp_only_npi_licensor_present":1,"blimp_only_npi_scope":1,"blimp_passive_1":1,"blimp_passive_2":1,"blimp_principle_A_c_command":1,"blimp_principle_A_case_1":1,"blimp_principle_A_case_2":1,"blimp_principle_A_domain_1":1,"blimp_principle_A_domain_2":1,"blimp_principle_A_domain_3":1,"blimp_principle_A_reconstruction":1,"blimp_regular_plural_subject_verb_agreement_1":1,"blimp_regular_plural_subject_verb_agreement_2":1,"blimp_sentential_negation_npi_licensor_present":1,"blimp_sentential_negation_npi_scope":1,"blimp_sentential_subject_island":1,"blimp_superlative_quantifiers_1":1,"blimp_superlative_quantifiers_2":1,"blimp_tough_vs_raising_1":1,"blimp_tough_vs_raising_2":1,"blimp_transitive":1,"blimp_wh_island":1,"blimp_wh_questions_object_gap":1,"blimp_wh_questions_subject_gap":1,"blimp_wh_questions_subject_gap_long_distance":1,"blimp_wh_vs_that_no_gap":1,"blimp_wh_vs_that_no_gap_long_distance":1,"blimp_wh_vs_that_with_gap":1,"blimp_wh_vs_that_with_gap_long_distance":1,"commonsense_qa_text":"Yaml","hellaswag":1,"piqa":1}},"rank_range":[13,15],"release_date":"2026-01-12","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/falcon-h1-tiny-r-90m/20260720T072155Z","source":"tiiuae/Falcon-H1-Tiny-R-90M","track":"instruction","updated_at":"2026-07-20T07:28:52.389064+00:00","vocab_size":32768},{"artifacts":{"result_file":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T154647Z-v1.1/results_v1.1_merged.json","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T154647Z-v1.1"},"benchmarks":{"arc_challenge":{"accuracy":0.2354948805460751,"chance":0.25,"ci":[-5.1222980659840704,1.251422070534695],"classification":"indistinguishable from chance","label":"ARC-Challenge","n_samples":1172,"score":-1.9340159271899877},"arc_easy":{"accuracy":0.3569023569023569,"chance":0.25,"ci":[11.560044893378228,16.835016835016834],"classification":"reliably above chance","label":"ARC-Easy","n_samples":2376,"score":14.253647586980918},"blimp":{"accuracy":0.772000496031746,"categories":{"Anaphor agreement":0.8959999999999999,"Argument structure":0.7603333333333333,"Binding":0.7562857142857142,"Control / raising":0.757,"Determiner-noun agreement":0.89825,"Ellipsis":0.7855,"Filler-gap":0.7295714285714286,"Irregular forms":0.9505,"Island effects":0.508875,"NPI licensing":0.7078571428571429,"Quantifiers":0.6639999999999999,"Subject-verb agreement":0.8498333333333333},"category_count":12,"chance":0.5,"ci":[53.51157457010582,54.73998561507939],"classification":"reliably above chance","label":"BLiMP","n_samples":70000,"score":54.4000992063492},"commonsense_qa":{"accuracy":0.2596232596232596,"chance":0.2,"ci":[4.484029484029484,10.62653562653563],"classification":"reliably above chance","label":"CommonsenseQA","n_samples":1221,"score":7.452907452907449},"hellaswag":{"accuracy":0.2731527584146584,"chance":0.25,"ci":[1.958109274380932,4.228905264555535],"classification":"reliably above chance","label":"HellaSwag","n_samples":10042,"score":3.087034455287789},"piqa":{"accuracy":0.5707290533188248,"chance":0.5,"ci":[9.681719260065288,18.50108813928181],"classification":"reliably above chance","label":"PIQA","n_samples":1838,"score":14.14581066376497}},"complete":true,"composite":{"ci":[19.071533266309793,21.225442487059897],"classification":"reliably above chance","score":20.26599982402355},"id":"slm-10m","name":"SLM 10M","params_m":9.96864,"protocol":{"bootstrap_replicates":2000,"bootstrap_seed":20260716,"harness":"lm-eval==0.4.12","harness_git_hash":null,"inference_config":{"batch_size":"auto","batch_sizes":[64],"bootstrap_iters":100000,"device":"cuda:0","fewshot_seed":1234,"gen_kwargs":{},"limit":null,"model":"hf","model_args":{"dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true},"model_dtype":"torch.bfloat16","model_num_parameters":9968640,"model_revision":"main","model_sha":"","numpy_seed":1234,"random_seed":1234,"tasks":["hellaswag","piqa","arc_easy","arc_challenge","commonsense_qa_text","blimp"],"torch_seed":1234,"use_cache":null,"v1_1_sources":{"rechecked_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T154341Z-v1.1/results_v1.1_merged.json","unchanged_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T154341Z-v1.1/results_v1.1_merged.json"}},"n_shot":{"arc_easy":0,"blimp":0,"blimp_adjunct_island":0,"blimp_anaphor_gender_agreement":0,"blimp_anaphor_number_agreement":0,"blimp_animate_subject_passive":0,"blimp_animate_subject_trans":0,"blimp_causative":0,"blimp_complex_NP_island":0,"blimp_coordinate_structure_constraint_complex_left_branch":0,"blimp_coordinate_structure_constraint_object_extraction":0,"blimp_determiner_noun_agreement_1":0,"blimp_determiner_noun_agreement_2":0,"blimp_determiner_noun_agreement_irregular_1":0,"blimp_determiner_noun_agreement_irregular_2":0,"blimp_determiner_noun_agreement_with_adj_2":0,"blimp_determiner_noun_agreement_with_adj_irregular_1":0,"blimp_determiner_noun_agreement_with_adj_irregular_2":0,"blimp_determiner_noun_agreement_with_adjective_1":0,"blimp_distractor_agreement_relational_noun":0,"blimp_distractor_agreement_relative_clause":0,"blimp_drop_argument":0,"blimp_ellipsis_n_bar_1":0,"blimp_ellipsis_n_bar_2":0,"blimp_existential_there_object_raising":0,"blimp_existential_there_quantifiers_1":0,"blimp_existential_there_quantifiers_2":0,"blimp_existential_there_subject_raising":0,"blimp_expletive_it_object_raising":0,"blimp_inchoative":0,"blimp_intransitive":0,"blimp_irregular_past_participle_adjectives":0,"blimp_irregular_past_participle_verbs":0,"blimp_irregular_plural_subject_verb_agreement_1":0,"blimp_irregular_plural_subject_verb_agreement_2":0,"blimp_left_branch_island_echo_question":0,"blimp_left_branch_island_simple_question":0,"blimp_matrix_question_npi_licensor_present":0,"blimp_npi_present_1":0,"blimp_npi_present_2":0,"blimp_only_npi_licensor_present":0,"blimp_only_npi_scope":0,"blimp_passive_1":0,"blimp_passive_2":0,"blimp_principle_A_c_command":0,"blimp_principle_A_case_1":0,"blimp_principle_A_case_2":0,"blimp_principle_A_domain_1":0,"blimp_principle_A_domain_2":0,"blimp_principle_A_domain_3":0,"blimp_principle_A_reconstruction":0,"blimp_regular_plural_subject_verb_agreement_1":0,"blimp_regular_plural_subject_verb_agreement_2":0,"blimp_sentential_negation_npi_licensor_present":0,"blimp_sentential_negation_npi_scope":0,"blimp_sentential_subject_island":0,"blimp_superlative_quantifiers_1":0,"blimp_superlative_quantifiers_2":0,"blimp_tough_vs_raising_1":0,"blimp_tough_vs_raising_2":0,"blimp_transitive":0,"blimp_wh_island":0,"blimp_wh_questions_object_gap":0,"blimp_wh_questions_subject_gap":0,"blimp_wh_questions_subject_gap_long_distance":0,"blimp_wh_vs_that_no_gap":0,"blimp_wh_vs_that_no_gap_long_distance":0,"blimp_wh_vs_that_with_gap":0,"blimp_wh_vs_that_with_gap_long_distance":0,"commonsense_qa":0,"hellaswag":0,"piqa":0},"suite":"slm-benchmark-v1.1","task_config":{"arc_challenge":{"dataset_name":"ARC-Challenge","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_challenge.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T153812Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_challenge","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"arc_easy":{"dataset_name":"ARC-Easy","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_easy.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_easy","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"blimp_adjunct_island":{"dataset_name":"adjunct_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/adjunct_island.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_adjunct_island","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_gender_agreement":{"dataset_name":"anaphor_gender_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_gender_agreement","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_number_agreement":{"dataset_name":"anaphor_number_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_number_agreement.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_number_agreement","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_passive":{"dataset_name":"animate_subject_passive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_passive.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_passive","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_trans":{"dataset_name":"animate_subject_trans","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_trans.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_trans","unsafe_code":false,"validation_split":"train"},"blimp_causative":{"dataset_name":"causative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/causative.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_causative","unsafe_code":false,"validation_split":"train"},"blimp_complex_NP_island":{"dataset_name":"complex_NP_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/complex_NP_island.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_complex_NP_island","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_complex_left_branch":{"dataset_name":"coordinate_structure_constraint_complex_left_branch","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_complex_left_branch","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_object_extraction":{"dataset_name":"coordinate_structure_constraint_object_extraction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_object_extraction","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_1":{"dataset_name":"determiner_noun_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_2":{"dataset_name":"determiner_noun_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_1":{"dataset_name":"determiner_noun_agreement_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_2":{"dataset_name":"determiner_noun_agreement_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_2":{"dataset_name":"determiner_noun_agreement_with_adj_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_1":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_2":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adjective_1":{"dataset_name":"determiner_noun_agreement_with_adjective_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adjective_1","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relational_noun":{"dataset_name":"distractor_agreement_relational_noun","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relational_noun","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relative_clause":{"dataset_name":"distractor_agreement_relative_clause","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relative_clause","unsafe_code":false,"validation_split":"train"},"blimp_drop_argument":{"dataset_name":"drop_argument","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/drop_argument.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_drop_argument","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_1":{"dataset_name":"ellipsis_n_bar_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_1","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_2":{"dataset_name":"ellipsis_n_bar_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_object_raising":{"dataset_name":"existential_there_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_object_raising.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_1":{"dataset_name":"existential_there_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_2":{"dataset_name":"existential_there_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_subject_raising":{"dataset_name":"existential_there_subject_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_subject_raising.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_subject_raising","unsafe_code":false,"validation_split":"train"},"blimp_expletive_it_object_raising":{"dataset_name":"expletive_it_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/expletive_it_object_raising.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_expletive_it_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_inchoative":{"dataset_name":"inchoative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/inchoative.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_inchoative","unsafe_code":false,"validation_split":"train"},"blimp_intransitive":{"dataset_name":"intransitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/intransitive.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_intransitive","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_adjectives":{"dataset_name":"irregular_past_participle_adjectives","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_adjectives","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_verbs":{"dataset_name":"irregular_past_participle_verbs","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_verbs","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_1":{"dataset_name":"irregular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_2":{"dataset_name":"irregular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_echo_question":{"dataset_name":"left_branch_island_echo_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_echo_question","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_simple_question":{"dataset_name":"left_branch_island_simple_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_simple_question","unsafe_code":false,"validation_split":"train"},"blimp_matrix_question_npi_licensor_present":{"dataset_name":"matrix_question_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_matrix_question_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_1":{"dataset_name":"npi_present_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_1.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_1","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_2":{"dataset_name":"npi_present_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_2.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_2","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_licensor_present":{"dataset_name":"only_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_scope":{"dataset_name":"only_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_passive_1":{"dataset_name":"passive_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_1.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_1","unsafe_code":false,"validation_split":"train"},"blimp_passive_2":{"dataset_name":"passive_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_2.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_c_command":{"dataset_name":"principle_A_c_command","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_c_command.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_c_command","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_1":{"dataset_name":"principle_A_case_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_1.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_2":{"dataset_name":"principle_A_case_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_2.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_1":{"dataset_name":"principle_A_domain_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_1.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_2":{"dataset_name":"principle_A_domain_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_2.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_3":{"dataset_name":"principle_A_domain_3","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_3.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_3","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_reconstruction":{"dataset_name":"principle_A_reconstruction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_reconstruction.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_reconstruction","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_1":{"dataset_name":"regular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_2":{"dataset_name":"regular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_licensor_present":{"dataset_name":"sentential_negation_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_scope":{"dataset_name":"sentential_negation_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_sentential_subject_island":{"dataset_name":"sentential_subject_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_subject_island.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_subject_island","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_1":{"dataset_name":"superlative_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_2":{"dataset_name":"superlative_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_1":{"dataset_name":"tough_vs_raising_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_1.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_1","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_2":{"dataset_name":"tough_vs_raising_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_2.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_2","unsafe_code":false,"validation_split":"train"},"blimp_transitive":{"dataset_name":"transitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/transitive.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_transitive","unsafe_code":false,"validation_split":"train"},"blimp_wh_island":{"dataset_name":"wh_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_island.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_island","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_object_gap":{"dataset_name":"wh_questions_object_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_object_gap.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_object_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap":{"dataset_name":"wh_questions_subject_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap_long_distance":{"dataset_name":"wh_questions_subject_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap":{"dataset_name":"wh_vs_that_no_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap_long_distance":{"dataset_name":"wh_vs_that_no_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap":{"dataset_name":"wh_vs_that_with_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap_long_distance":{"dataset_name":"wh_vs_that_with_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap_long_distance","unsafe_code":false,"validation_split":"train"},"commonsense_qa_text":{"dataset_path":"tau/commonsense_qa","description":"","doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/config/tasks/commonsense_qa_text.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T153812Z/tokenizer_compat","trust_remote_code":true},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"commonsense_qa_text","training_split":"train","unsafe_code":false,"validation_split":"validation"},"hellaswag":{"dataset_path":"Rowan/hellaswag","description":"","doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_config":{"doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":"<function process_docs at 0x7e9023f9ccc0>","sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","process_docs":"def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\\n    def _process_doc(doc):\\n        ctx = doc[\\"ctx_a\\"] + \\" \\" + doc[\\"ctx_b\\"].capitalize()\\n        out_doc = {\\n            \\"query\\": preprocess(doc[\\"activity_label\\"] + \\": \\" + ctx),\\n            \\"choices\\": [preprocess(ending) for ending in doc[\\"endings\\"]],\\n            \\"gold\\": int(doc[\\"label\\"]),\\n        }\\n        return out_doc\\n\\n    return dataset.map(_process_doc)\\n","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"hellaswag","training_split":"train","unsafe_code":false,"validation_split":"validation"},"piqa":{"dataset_path":"baber/piqa","description":"","doc_to_choice":"{{[sol1, sol2]}}","doc_to_decontamination_query":"goal","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{[sol1, sol2]}}","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/piqa/piqa.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"piqa","training_split":"train","unsafe_code":false,"validation_split":"validation"}},"task_versions":{"arc_challenge":1,"arc_easy":1,"blimp":"2.0","blimp_adjunct_island":1,"blimp_anaphor_gender_agreement":1,"blimp_anaphor_number_agreement":1,"blimp_animate_subject_passive":1,"blimp_animate_subject_trans":1,"blimp_causative":1,"blimp_complex_NP_island":1,"blimp_coordinate_structure_constraint_complex_left_branch":1,"blimp_coordinate_structure_constraint_object_extraction":1,"blimp_determiner_noun_agreement_1":1,"blimp_determiner_noun_agreement_2":1,"blimp_determiner_noun_agreement_irregular_1":1,"blimp_determiner_noun_agreement_irregular_2":1,"blimp_determiner_noun_agreement_with_adj_2":1,"blimp_determiner_noun_agreement_with_adj_irregular_1":1,"blimp_determiner_noun_agreement_with_adj_irregular_2":1,"blimp_determiner_noun_agreement_with_adjective_1":1,"blimp_distractor_agreement_relational_noun":1,"blimp_distractor_agreement_relative_clause":1,"blimp_drop_argument":1,"blimp_ellipsis_n_bar_1":1,"blimp_ellipsis_n_bar_2":1,"blimp_existential_there_object_raising":1,"blimp_existential_there_quantifiers_1":1,"blimp_existential_there_quantifiers_2":1,"blimp_existential_there_subject_raising":1,"blimp_expletive_it_object_raising":1,"blimp_inchoative":1,"blimp_intransitive":1,"blimp_irregular_past_participle_adjectives":1,"blimp_irregular_past_participle_verbs":1,"blimp_irregular_plural_subject_verb_agreement_1":1,"blimp_irregular_plural_subject_verb_agreement_2":1,"blimp_left_branch_island_echo_question":1,"blimp_left_branch_island_simple_question":1,"blimp_matrix_question_npi_licensor_present":1,"blimp_npi_present_1":1,"blimp_npi_present_2":1,"blimp_only_npi_licensor_present":1,"blimp_only_npi_scope":1,"blimp_passive_1":1,"blimp_passive_2":1,"blimp_principle_A_c_command":1,"blimp_principle_A_case_1":1,"blimp_principle_A_case_2":1,"blimp_principle_A_domain_1":1,"blimp_principle_A_domain_2":1,"blimp_principle_A_domain_3":1,"blimp_principle_A_reconstruction":1,"blimp_regular_plural_subject_verb_agreement_1":1,"blimp_regular_plural_subject_verb_agreement_2":1,"blimp_sentential_negation_npi_licensor_present":1,"blimp_sentential_negation_npi_scope":1,"blimp_sentential_subject_island":1,"blimp_superlative_quantifiers_1":1,"blimp_superlative_quantifiers_2":1,"blimp_tough_vs_raising_1":1,"blimp_tough_vs_raising_2":1,"blimp_transitive":1,"blimp_wh_island":1,"blimp_wh_questions_object_gap":1,"blimp_wh_questions_subject_gap":1,"blimp_wh_questions_subject_gap_long_distance":1,"blimp_wh_vs_that_no_gap":1,"blimp_wh_vs_that_no_gap_long_distance":1,"blimp_wh_vs_that_with_gap":1,"blimp_wh_vs_that_with_gap_long_distance":1,"commonsense_qa_text":"Yaml","hellaswag":1,"piqa":1}},"rank_range":[13,15],"release_date":"2026-06-13","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T154647Z-v1.1","source":"liodon-ai/slm-10m","track":"base","updated_at":"2026-07-16T15:47:03.814706+00:00","vocab_size":8192},{"artifacts":{"result_file":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/pythia-160m/20260716T154647Z-v1.1/results_v1.1_merged.json","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/pythia-160m/20260716T154647Z-v1.1"},"benchmarks":{"arc_challenge":{"accuracy":0.23037542662116042,"chance":0.25,"ci":[-5.688282138794084,0.5688282138794095],"classification":"indistinguishable from chance","label":"ARC-Challenge","n_samples":1172,"score":-2.616609783845277},"arc_easy":{"accuracy":0.3686868686868687,"chance":0.25,"ci":[13.18742985409652,18.350168350168353],"classification":"reliably above chance","label":"ARC-Easy","n_samples":2376,"score":15.824915824915822},"blimp":{"accuracy":0.7286577050264551,"categories":{"Anaphor agreement":0.9245000000000001,"Argument structure":0.7542222222222222,"Binding":0.7448571428571429,"Control / raising":0.7462,"Determiner-noun agreement":0.826125,"Ellipsis":0.759,"Filler-gap":0.6882857142857144,"Irregular forms":0.723,"Island effects":0.631,"NPI licensing":0.5852857142857142,"Quantifiers":0.63075,"Subject-verb agreement":0.7306666666666667},"category_count":12,"chance":0.5,"ci":[44.78496957671956,46.238928736772515],"classification":"reliably above chance","label":"BLiMP","n_samples":70000,"score":45.73154100529102},"commonsense_qa":{"accuracy":0.2932022932022932,"chance":0.2,"ci":[8.37428337428337,14.823914823914825],"classification":"reliably above chance","label":"CommonsenseQA","n_samples":1221,"score":11.650286650286647},"hellaswag":{"accuracy":0.30501892053375823,"chance":0.25,"ci":[6.167430126800774,8.504282015534756],"classification":"reliably above chance","label":"HellaSwag","n_samples":10042,"score":7.335856071167764},"piqa":{"accuracy":0.5843307943416758,"chance":0.5,"ci":[12.51360174102285,21.218715995647443],"classification":"reliably above chance","label":"PIQA","n_samples":1838,"score":16.866158868335155}},"complete":true,"composite":{"ci":[19.111437730092515,21.23647682872686],"classification":"reliably above chance","score":20.2119024553024},"id":"pythia-160m","name":"Pythia 160M","params_m":160,"protocol":{"bootstrap_replicates":2000,"bootstrap_seed":20260716,"harness":"lm-eval==0.4.12","harness_git_hash":null,"inference_config":{"batch_size":"auto","batch_sizes":[64],"bootstrap_iters":100000,"device":"cuda:0","fewshot_seed":1234,"gen_kwargs":{},"limit":null,"model":"hf","model_args":{"dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false},"model_dtype":"torch.bfloat16","model_num_parameters":162322944,"model_revision":"main","model_sha":"","numpy_seed":1234,"random_seed":1234,"tasks":["hellaswag","piqa","arc_easy","arc_challenge","commonsense_qa_text","blimp"],"torch_seed":1234,"use_cache":null,"v1_1_sources":{"rechecked_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/pythia-160m/20260716T154341Z-v1.1/results_v1.1_merged.json","unchanged_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/pythia-160m/20260716T154341Z-v1.1/results_v1.1_merged.json"}},"n_shot":{"arc_easy":0,"blimp":0,"blimp_adjunct_island":0,"blimp_anaphor_gender_agreement":0,"blimp_anaphor_number_agreement":0,"blimp_animate_subject_passive":0,"blimp_animate_subject_trans":0,"blimp_causative":0,"blimp_complex_NP_island":0,"blimp_coordinate_structure_constraint_complex_left_branch":0,"blimp_coordinate_structure_constraint_object_extraction":0,"blimp_determiner_noun_agreement_1":0,"blimp_determiner_noun_agreement_2":0,"blimp_determiner_noun_agreement_irregular_1":0,"blimp_determiner_noun_agreement_irregular_2":0,"blimp_determiner_noun_agreement_with_adj_2":0,"blimp_determiner_noun_agreement_with_adj_irregular_1":0,"blimp_determiner_noun_agreement_with_adj_irregular_2":0,"blimp_determiner_noun_agreement_with_adjective_1":0,"blimp_distractor_agreement_relational_noun":0,"blimp_distractor_agreement_relative_clause":0,"blimp_drop_argument":0,"blimp_ellipsis_n_bar_1":0,"blimp_ellipsis_n_bar_2":0,"blimp_existential_there_object_raising":0,"blimp_existential_there_quantifiers_1":0,"blimp_existential_there_quantifiers_2":0,"blimp_existential_there_subject_raising":0,"blimp_expletive_it_object_raising":0,"blimp_inchoative":0,"blimp_intransitive":0,"blimp_irregular_past_participle_adjectives":0,"blimp_irregular_past_participle_verbs":0,"blimp_irregular_plural_subject_verb_agreement_1":0,"blimp_irregular_plural_subject_verb_agreement_2":0,"blimp_left_branch_island_echo_question":0,"blimp_left_branch_island_simple_question":0,"blimp_matrix_question_npi_licensor_present":0,"blimp_npi_present_1":0,"blimp_npi_present_2":0,"blimp_only_npi_licensor_present":0,"blimp_only_npi_scope":0,"blimp_passive_1":0,"blimp_passive_2":0,"blimp_principle_A_c_command":0,"blimp_principle_A_case_1":0,"blimp_principle_A_case_2":0,"blimp_principle_A_domain_1":0,"blimp_principle_A_domain_2":0,"blimp_principle_A_domain_3":0,"blimp_principle_A_reconstruction":0,"blimp_regular_plural_subject_verb_agreement_1":0,"blimp_regular_plural_subject_verb_agreement_2":0,"blimp_sentential_negation_npi_licensor_present":0,"blimp_sentential_negation_npi_scope":0,"blimp_sentential_subject_island":0,"blimp_superlative_quantifiers_1":0,"blimp_superlative_quantifiers_2":0,"blimp_tough_vs_raising_1":0,"blimp_tough_vs_raising_2":0,"blimp_transitive":0,"blimp_wh_island":0,"blimp_wh_questions_object_gap":0,"blimp_wh_questions_subject_gap":0,"blimp_wh_questions_subject_gap_long_distance":0,"blimp_wh_vs_that_no_gap":0,"blimp_wh_vs_that_no_gap_long_distance":0,"blimp_wh_vs_that_with_gap":0,"blimp_wh_vs_that_with_gap_long_distance":0,"commonsense_qa":0,"hellaswag":0,"piqa":0},"suite":"slm-benchmark-v1.1","task_config":{"arc_challenge":{"dataset_name":"ARC-Challenge","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_challenge.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_challenge","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"arc_easy":{"dataset_name":"ARC-Easy","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_easy.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_easy","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"blimp_adjunct_island":{"dataset_name":"adjunct_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/adjunct_island.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_adjunct_island","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_gender_agreement":{"dataset_name":"anaphor_gender_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_gender_agreement","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_number_agreement":{"dataset_name":"anaphor_number_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_number_agreement.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_number_agreement","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_passive":{"dataset_name":"animate_subject_passive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_passive.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_passive","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_trans":{"dataset_name":"animate_subject_trans","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_trans.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_trans","unsafe_code":false,"validation_split":"train"},"blimp_causative":{"dataset_name":"causative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/causative.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_causative","unsafe_code":false,"validation_split":"train"},"blimp_complex_NP_island":{"dataset_name":"complex_NP_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/complex_NP_island.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_complex_NP_island","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_complex_left_branch":{"dataset_name":"coordinate_structure_constraint_complex_left_branch","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_complex_left_branch","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_object_extraction":{"dataset_name":"coordinate_structure_constraint_object_extraction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_object_extraction","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_1":{"dataset_name":"determiner_noun_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_2":{"dataset_name":"determiner_noun_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_1":{"dataset_name":"determiner_noun_agreement_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_2":{"dataset_name":"determiner_noun_agreement_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_2":{"dataset_name":"determiner_noun_agreement_with_adj_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_1":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_2":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adjective_1":{"dataset_name":"determiner_noun_agreement_with_adjective_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adjective_1","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relational_noun":{"dataset_name":"distractor_agreement_relational_noun","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relational_noun","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relative_clause":{"dataset_name":"distractor_agreement_relative_clause","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relative_clause","unsafe_code":false,"validation_split":"train"},"blimp_drop_argument":{"dataset_name":"drop_argument","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/drop_argument.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_drop_argument","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_1":{"dataset_name":"ellipsis_n_bar_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_1","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_2":{"dataset_name":"ellipsis_n_bar_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_object_raising":{"dataset_name":"existential_there_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_object_raising.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_1":{"dataset_name":"existential_there_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_2":{"dataset_name":"existential_there_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_subject_raising":{"dataset_name":"existential_there_subject_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_subject_raising.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_subject_raising","unsafe_code":false,"validation_split":"train"},"blimp_expletive_it_object_raising":{"dataset_name":"expletive_it_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/expletive_it_object_raising.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_expletive_it_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_inchoative":{"dataset_name":"inchoative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/inchoative.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_inchoative","unsafe_code":false,"validation_split":"train"},"blimp_intransitive":{"dataset_name":"intransitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/intransitive.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_intransitive","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_adjectives":{"dataset_name":"irregular_past_participle_adjectives","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_adjectives","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_verbs":{"dataset_name":"irregular_past_participle_verbs","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_verbs","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_1":{"dataset_name":"irregular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_2":{"dataset_name":"irregular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_echo_question":{"dataset_name":"left_branch_island_echo_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_echo_question","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_simple_question":{"dataset_name":"left_branch_island_simple_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_simple_question","unsafe_code":false,"validation_split":"train"},"blimp_matrix_question_npi_licensor_present":{"dataset_name":"matrix_question_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_matrix_question_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_1":{"dataset_name":"npi_present_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_1","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_2":{"dataset_name":"npi_present_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_2","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_licensor_present":{"dataset_name":"only_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_scope":{"dataset_name":"only_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_passive_1":{"dataset_name":"passive_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_1","unsafe_code":false,"validation_split":"train"},"blimp_passive_2":{"dataset_name":"passive_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_c_command":{"dataset_name":"principle_A_c_command","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_c_command.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_c_command","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_1":{"dataset_name":"principle_A_case_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_2":{"dataset_name":"principle_A_case_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_1":{"dataset_name":"principle_A_domain_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_2":{"dataset_name":"principle_A_domain_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_3":{"dataset_name":"principle_A_domain_3","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_3.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_3","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_reconstruction":{"dataset_name":"principle_A_reconstruction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_reconstruction.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_reconstruction","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_1":{"dataset_name":"regular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_2":{"dataset_name":"regular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_licensor_present":{"dataset_name":"sentential_negation_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_scope":{"dataset_name":"sentential_negation_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_sentential_subject_island":{"dataset_name":"sentential_subject_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_subject_island.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_subject_island","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_1":{"dataset_name":"superlative_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_2":{"dataset_name":"superlative_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_1":{"dataset_name":"tough_vs_raising_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_1","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_2":{"dataset_name":"tough_vs_raising_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_2","unsafe_code":false,"validation_split":"train"},"blimp_transitive":{"dataset_name":"transitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/transitive.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_transitive","unsafe_code":false,"validation_split":"train"},"blimp_wh_island":{"dataset_name":"wh_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_island.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_island","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_object_gap":{"dataset_name":"wh_questions_object_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_object_gap.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_object_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap":{"dataset_name":"wh_questions_subject_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap_long_distance":{"dataset_name":"wh_questions_subject_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap":{"dataset_name":"wh_vs_that_no_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap_long_distance":{"dataset_name":"wh_vs_that_no_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap":{"dataset_name":"wh_vs_that_with_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap_long_distance":{"dataset_name":"wh_vs_that_with_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap_long_distance","unsafe_code":false,"validation_split":"train"},"commonsense_qa_text":{"dataset_path":"tau/commonsense_qa","description":"","doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/config/tasks/commonsense_qa_text.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"commonsense_qa_text","training_split":"train","unsafe_code":false,"validation_split":"validation"},"hellaswag":{"dataset_path":"Rowan/hellaswag","description":"","doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_config":{"doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":"<function process_docs at 0x71da3d32f9c0>","sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","process_docs":"def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\\n    def _process_doc(doc):\\n        ctx = doc[\\"ctx_a\\"] + \\" \\" + doc[\\"ctx_b\\"].capitalize()\\n        out_doc = {\\n            \\"query\\": preprocess(doc[\\"activity_label\\"] + \\": \\" + ctx),\\n            \\"choices\\": [preprocess(ending) for ending in doc[\\"endings\\"]],\\n            \\"gold\\": int(doc[\\"label\\"]),\\n        }\\n        return out_doc\\n\\n    return dataset.map(_process_doc)\\n","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"hellaswag","training_split":"train","unsafe_code":false,"validation_split":"validation"},"piqa":{"dataset_path":"baber/piqa","description":"","doc_to_choice":"{{[sol1, sol2]}}","doc_to_decontamination_query":"goal","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{[sol1, sol2]}}","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/piqa/piqa.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"piqa","training_split":"train","unsafe_code":false,"validation_split":"validation"}},"task_versions":{"arc_challenge":1,"arc_easy":1,"blimp":"2.0","blimp_adjunct_island":1,"blimp_anaphor_gender_agreement":1,"blimp_anaphor_number_agreement":1,"blimp_animate_subject_passive":1,"blimp_animate_subject_trans":1,"blimp_causative":1,"blimp_complex_NP_island":1,"blimp_coordinate_structure_constraint_complex_left_branch":1,"blimp_coordinate_structure_constraint_object_extraction":1,"blimp_determiner_noun_agreement_1":1,"blimp_determiner_noun_agreement_2":1,"blimp_determiner_noun_agreement_irregular_1":1,"blimp_determiner_noun_agreement_irregular_2":1,"blimp_determiner_noun_agreement_with_adj_2":1,"blimp_determiner_noun_agreement_with_adj_irregular_1":1,"blimp_determiner_noun_agreement_with_adj_irregular_2":1,"blimp_determiner_noun_agreement_with_adjective_1":1,"blimp_distractor_agreement_relational_noun":1,"blimp_distractor_agreement_relative_clause":1,"blimp_drop_argument":1,"blimp_ellipsis_n_bar_1":1,"blimp_ellipsis_n_bar_2":1,"blimp_existential_there_object_raising":1,"blimp_existential_there_quantifiers_1":1,"blimp_existential_there_quantifiers_2":1,"blimp_existential_there_subject_raising":1,"blimp_expletive_it_object_raising":1,"blimp_inchoative":1,"blimp_intransitive":1,"blimp_irregular_past_participle_adjectives":1,"blimp_irregular_past_participle_verbs":1,"blimp_irregular_plural_subject_verb_agreement_1":1,"blimp_irregular_plural_subject_verb_agreement_2":1,"blimp_left_branch_island_echo_question":1,"blimp_left_branch_island_simple_question":1,"blimp_matrix_question_npi_licensor_present":1,"blimp_npi_present_1":1,"blimp_npi_present_2":1,"blimp_only_npi_licensor_present":1,"blimp_only_npi_scope":1,"blimp_passive_1":1,"blimp_passive_2":1,"blimp_principle_A_c_command":1,"blimp_principle_A_case_1":1,"blimp_principle_A_case_2":1,"blimp_principle_A_domain_1":1,"blimp_principle_A_domain_2":1,"blimp_principle_A_domain_3":1,"blimp_principle_A_reconstruction":1,"blimp_regular_plural_subject_verb_agreement_1":1,"blimp_regular_plural_subject_verb_agreement_2":1,"blimp_sentential_negation_npi_licensor_present":1,"blimp_sentential_negation_npi_scope":1,"blimp_sentential_subject_island":1,"blimp_superlative_quantifiers_1":1,"blimp_superlative_quantifiers_2":1,"blimp_tough_vs_raising_1":1,"blimp_tough_vs_raising_2":1,"blimp_transitive":1,"blimp_wh_island":1,"blimp_wh_questions_object_gap":1,"blimp_wh_questions_subject_gap":1,"blimp_wh_questions_subject_gap_long_distance":1,"blimp_wh_vs_that_no_gap":1,"blimp_wh_vs_that_no_gap_long_distance":1,"blimp_wh_vs_that_with_gap":1,"blimp_wh_vs_that_with_gap_long_distance":1,"commonsense_qa_text":"Yaml","hellaswag":1,"piqa":1}},"rank_range":[13,16],"release_date":"2023-02-08","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/pythia-160m/20260716T154647Z-v1.1","source":"EleutherAI/pythia-160m","track":"base","updated_at":"2026-07-16T15:46:58.993182+00:00","vocab_size":50304},{"artifacts":{"result_file":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt-s2-5m/20260716T154647Z-v1.1/results_v1.1_merged.json","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt-s2-5m/20260716T154647Z-v1.1"},"benchmarks":{"arc_challenge":{"accuracy":0.22184300341296928,"chance":0.25,"ci":[-6.939704209328782,-0.6825938566552892],"classification":"below chance","label":"ARC-Challenge","n_samples":1172,"score":-3.754266211604096},"arc_easy":{"accuracy":0.335016835016835,"chance":0.25,"ci":[8.585858585858587,13.916947250280586],"classification":"reliably above chance","label":"ARC-Easy","n_samples":2376,"score":11.335578002244665},"blimp":{"accuracy":0.7732386243386244,"categories":{"Anaphor agreement":0.9405,"Argument structure":0.7917777777777779,"Binding":0.732857142857143,"Control / raising":0.7488,"Determiner-noun agreement":0.902375,"Ellipsis":0.7815000000000001,"Filler-gap":0.7285714285714285,"Irregular forms":0.915,"Island effects":0.538875,"NPI licensing":0.7108571428571429,"Quantifiers":0.63225,"Subject-verb agreement":0.8555000000000001},"category_count":12,"chance":0.5,"ci":[53.86334275793652,55.039749007936514],"classification":"reliably above chance","label":"BLiMP","n_samples":70000,"score":54.64772486772489},"commonsense_qa":{"accuracy":0.24815724815724816,"chance":0.2,"ci":[3.1505937755937747,9.193284193284189],"classification":"reliably above chance","label":"CommonsenseQA","n_samples":1221,"score":6.019656019656018},"hellaswag":{"accuracy":0.27544313881696875,"chance":0.25,"ci":[2.2638252672110504,4.5083980614751376],"classification":"reliably above chance","label":"HellaSwag","n_samples":10042,"score":3.392418508929166},"piqa":{"accuracy":0.5647442872687704,"chance":0.5,"ci":[8.26985854189337,17.41022850924918],"classification":"reliably above chance","label":"PIQA","n_samples":1838,"score":12.94885745375407}},"complete":true,"composite":{"ci":[18.15426094733613,20.273369388072773],"classification":"reliably above chance","score":19.279532451183385},"id":"gpt-s2-5m","name":"GPT-S2 5M","params_m":5.384,"protocol":{"bootstrap_replicates":2000,"bootstrap_seed":20260716,"harness":"lm-eval==0.4.12","harness_git_hash":null,"inference_config":{"batch_size":"auto","batch_sizes":[64],"bootstrap_iters":100000,"device":"cuda:0","fewshot_seed":1234,"gen_kwargs":{},"limit":null,"model":"hf","model_args":{"dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true},"model_dtype":"torch.bfloat16","model_num_parameters":5384258,"model_revision":"main","model_sha":"","numpy_seed":1234,"random_seed":1234,"tasks":["hellaswag","piqa","arc_easy","arc_challenge","commonsense_qa_text","blimp"],"torch_seed":1234,"use_cache":null,"v1_1_sources":{"rechecked_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt-s2-5m/20260716T154341Z-v1.1/results_v1.1_merged.json","unchanged_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt-s2-5m/20260716T154341Z-v1.1/results_v1.1_merged.json"}},"n_shot":{"arc_easy":0,"blimp":0,"blimp_adjunct_island":0,"blimp_anaphor_gender_agreement":0,"blimp_anaphor_number_agreement":0,"blimp_animate_subject_passive":0,"blimp_animate_subject_trans":0,"blimp_causative":0,"blimp_complex_NP_island":0,"blimp_coordinate_structure_constraint_complex_left_branch":0,"blimp_coordinate_structure_constraint_object_extraction":0,"blimp_determiner_noun_agreement_1":0,"blimp_determiner_noun_agreement_2":0,"blimp_determiner_noun_agreement_irregular_1":0,"blimp_determiner_noun_agreement_irregular_2":0,"blimp_determiner_noun_agreement_with_adj_2":0,"blimp_determiner_noun_agreement_with_adj_irregular_1":0,"blimp_determiner_noun_agreement_with_adj_irregular_2":0,"blimp_determiner_noun_agreement_with_adjective_1":0,"blimp_distractor_agreement_relational_noun":0,"blimp_distractor_agreement_relative_clause":0,"blimp_drop_argument":0,"blimp_ellipsis_n_bar_1":0,"blimp_ellipsis_n_bar_2":0,"blimp_existential_there_object_raising":0,"blimp_existential_there_quantifiers_1":0,"blimp_existential_there_quantifiers_2":0,"blimp_existential_there_subject_raising":0,"blimp_expletive_it_object_raising":0,"blimp_inchoative":0,"blimp_intransitive":0,"blimp_irregular_past_participle_adjectives":0,"blimp_irregular_past_participle_verbs":0,"blimp_irregular_plural_subject_verb_agreement_1":0,"blimp_irregular_plural_subject_verb_agreement_2":0,"blimp_left_branch_island_echo_question":0,"blimp_left_branch_island_simple_question":0,"blimp_matrix_question_npi_licensor_present":0,"blimp_npi_present_1":0,"blimp_npi_present_2":0,"blimp_only_npi_licensor_present":0,"blimp_only_npi_scope":0,"blimp_passive_1":0,"blimp_passive_2":0,"blimp_principle_A_c_command":0,"blimp_principle_A_case_1":0,"blimp_principle_A_case_2":0,"blimp_principle_A_domain_1":0,"blimp_principle_A_domain_2":0,"blimp_principle_A_domain_3":0,"blimp_principle_A_reconstruction":0,"blimp_regular_plural_subject_verb_agreement_1":0,"blimp_regular_plural_subject_verb_agreement_2":0,"blimp_sentential_negation_npi_licensor_present":0,"blimp_sentential_negation_npi_scope":0,"blimp_sentential_subject_island":0,"blimp_superlative_quantifiers_1":0,"blimp_superlative_quantifiers_2":0,"blimp_tough_vs_raising_1":0,"blimp_tough_vs_raising_2":0,"blimp_transitive":0,"blimp_wh_island":0,"blimp_wh_questions_object_gap":0,"blimp_wh_questions_subject_gap":0,"blimp_wh_questions_subject_gap_long_distance":0,"blimp_wh_vs_that_no_gap":0,"blimp_wh_vs_that_no_gap_long_distance":0,"blimp_wh_vs_that_with_gap":0,"blimp_wh_vs_that_with_gap_long_distance":0,"commonsense_qa":0,"hellaswag":0,"piqa":0},"suite":"slm-benchmark-v1.1","task_config":{"arc_challenge":{"dataset_name":"ARC-Challenge","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_challenge.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_challenge","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"arc_easy":{"dataset_name":"ARC-Easy","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_easy.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_easy","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"blimp_adjunct_island":{"dataset_name":"adjunct_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/adjunct_island.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_adjunct_island","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_gender_agreement":{"dataset_name":"anaphor_gender_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_gender_agreement","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_number_agreement":{"dataset_name":"anaphor_number_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_number_agreement.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_number_agreement","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_passive":{"dataset_name":"animate_subject_passive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_passive.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_passive","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_trans":{"dataset_name":"animate_subject_trans","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_trans.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_trans","unsafe_code":false,"validation_split":"train"},"blimp_causative":{"dataset_name":"causative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/causative.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_causative","unsafe_code":false,"validation_split":"train"},"blimp_complex_NP_island":{"dataset_name":"complex_NP_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/complex_NP_island.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_complex_NP_island","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_complex_left_branch":{"dataset_name":"coordinate_structure_constraint_complex_left_branch","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_complex_left_branch","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_object_extraction":{"dataset_name":"coordinate_structure_constraint_object_extraction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_object_extraction","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_1":{"dataset_name":"determiner_noun_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_2":{"dataset_name":"determiner_noun_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_1":{"dataset_name":"determiner_noun_agreement_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_2":{"dataset_name":"determiner_noun_agreement_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_2":{"dataset_name":"determiner_noun_agreement_with_adj_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_1":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_2":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adjective_1":{"dataset_name":"determiner_noun_agreement_with_adjective_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adjective_1","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relational_noun":{"dataset_name":"distractor_agreement_relational_noun","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relational_noun","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relative_clause":{"dataset_name":"distractor_agreement_relative_clause","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relative_clause","unsafe_code":false,"validation_split":"train"},"blimp_drop_argument":{"dataset_name":"drop_argument","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/drop_argument.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_drop_argument","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_1":{"dataset_name":"ellipsis_n_bar_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_1","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_2":{"dataset_name":"ellipsis_n_bar_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_object_raising":{"dataset_name":"existential_there_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_object_raising.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_1":{"dataset_name":"existential_there_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_2":{"dataset_name":"existential_there_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_subject_raising":{"dataset_name":"existential_there_subject_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_subject_raising.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_subject_raising","unsafe_code":false,"validation_split":"train"},"blimp_expletive_it_object_raising":{"dataset_name":"expletive_it_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/expletive_it_object_raising.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_expletive_it_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_inchoative":{"dataset_name":"inchoative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/inchoative.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_inchoative","unsafe_code":false,"validation_split":"train"},"blimp_intransitive":{"dataset_name":"intransitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/intransitive.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_intransitive","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_adjectives":{"dataset_name":"irregular_past_participle_adjectives","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_adjectives","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_verbs":{"dataset_name":"irregular_past_participle_verbs","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_verbs","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_1":{"dataset_name":"irregular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_2":{"dataset_name":"irregular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_echo_question":{"dataset_name":"left_branch_island_echo_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_echo_question","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_simple_question":{"dataset_name":"left_branch_island_simple_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_simple_question","unsafe_code":false,"validation_split":"train"},"blimp_matrix_question_npi_licensor_present":{"dataset_name":"matrix_question_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_matrix_question_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_1":{"dataset_name":"npi_present_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_1","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_2":{"dataset_name":"npi_present_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_2","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_licensor_present":{"dataset_name":"only_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_scope":{"dataset_name":"only_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_passive_1":{"dataset_name":"passive_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_1","unsafe_code":false,"validation_split":"train"},"blimp_passive_2":{"dataset_name":"passive_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_c_command":{"dataset_name":"principle_A_c_command","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_c_command.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_c_command","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_1":{"dataset_name":"principle_A_case_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_2":{"dataset_name":"principle_A_case_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_1":{"dataset_name":"principle_A_domain_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_2":{"dataset_name":"principle_A_domain_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_3":{"dataset_name":"principle_A_domain_3","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_3.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_3","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_reconstruction":{"dataset_name":"principle_A_reconstruction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_reconstruction.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_reconstruction","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_1":{"dataset_name":"regular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_2":{"dataset_name":"regular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_licensor_present":{"dataset_name":"sentential_negation_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_scope":{"dataset_name":"sentential_negation_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_sentential_subject_island":{"dataset_name":"sentential_subject_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_subject_island.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_subject_island","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_1":{"dataset_name":"superlative_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_2":{"dataset_name":"superlative_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_1":{"dataset_name":"tough_vs_raising_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_1","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_2":{"dataset_name":"tough_vs_raising_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_2","unsafe_code":false,"validation_split":"train"},"blimp_transitive":{"dataset_name":"transitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/transitive.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_transitive","unsafe_code":false,"validation_split":"train"},"blimp_wh_island":{"dataset_name":"wh_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_island.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_island","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_object_gap":{"dataset_name":"wh_questions_object_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_object_gap.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_object_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap":{"dataset_name":"wh_questions_subject_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap_long_distance":{"dataset_name":"wh_questions_subject_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap":{"dataset_name":"wh_vs_that_no_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap_long_distance":{"dataset_name":"wh_vs_that_no_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap":{"dataset_name":"wh_vs_that_with_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap_long_distance":{"dataset_name":"wh_vs_that_with_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap_long_distance","unsafe_code":false,"validation_split":"train"},"commonsense_qa_text":{"dataset_path":"tau/commonsense_qa","description":"","doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/config/tasks/commonsense_qa_text.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"commonsense_qa_text","training_split":"train","unsafe_code":false,"validation_split":"validation"},"hellaswag":{"dataset_path":"Rowan/hellaswag","description":"","doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_config":{"doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":"<function process_docs at 0x7891fe910ae0>","sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","process_docs":"def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\\n    def _process_doc(doc):\\n        ctx = doc[\\"ctx_a\\"] + \\" \\" + doc[\\"ctx_b\\"].capitalize()\\n        out_doc = {\\n            \\"query\\": preprocess(doc[\\"activity_label\\"] + \\": \\" + ctx),\\n            \\"choices\\": [preprocess(ending) for ending in doc[\\"endings\\"]],\\n            \\"gold\\": int(doc[\\"label\\"]),\\n        }\\n        return out_doc\\n\\n    return dataset.map(_process_doc)\\n","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"hellaswag","training_split":"train","unsafe_code":false,"validation_split":"validation"},"piqa":{"dataset_path":"baber/piqa","description":"","doc_to_choice":"{{[sol1, sol2]}}","doc_to_decontamination_query":"goal","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{[sol1, sol2]}}","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/piqa/piqa.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"piqa","training_split":"train","unsafe_code":false,"validation_split":"validation"}},"task_versions":{"arc_challenge":1,"arc_easy":1,"blimp":"2.0","blimp_adjunct_island":1,"blimp_anaphor_gender_agreement":1,"blimp_anaphor_number_agreement":1,"blimp_animate_subject_passive":1,"blimp_animate_subject_trans":1,"blimp_causative":1,"blimp_complex_NP_island":1,"blimp_coordinate_structure_constraint_complex_left_branch":1,"blimp_coordinate_structure_constraint_object_extraction":1,"blimp_determiner_noun_agreement_1":1,"blimp_determiner_noun_agreement_2":1,"blimp_determiner_noun_agreement_irregular_1":1,"blimp_determiner_noun_agreement_irregular_2":1,"blimp_determiner_noun_agreement_with_adj_2":1,"blimp_determiner_noun_agreement_with_adj_irregular_1":1,"blimp_determiner_noun_agreement_with_adj_irregular_2":1,"blimp_determiner_noun_agreement_with_adjective_1":1,"blimp_distractor_agreement_relational_noun":1,"blimp_distractor_agreement_relative_clause":1,"blimp_drop_argument":1,"blimp_ellipsis_n_bar_1":1,"blimp_ellipsis_n_bar_2":1,"blimp_existential_there_object_raising":1,"blimp_existential_there_quantifiers_1":1,"blimp_existential_there_quantifiers_2":1,"blimp_existential_there_subject_raising":1,"blimp_expletive_it_object_raising":1,"blimp_inchoative":1,"blimp_intransitive":1,"blimp_irregular_past_participle_adjectives":1,"blimp_irregular_past_participle_verbs":1,"blimp_irregular_plural_subject_verb_agreement_1":1,"blimp_irregular_plural_subject_verb_agreement_2":1,"blimp_left_branch_island_echo_question":1,"blimp_left_branch_island_simple_question":1,"blimp_matrix_question_npi_licensor_present":1,"blimp_npi_present_1":1,"blimp_npi_present_2":1,"blimp_only_npi_licensor_present":1,"blimp_only_npi_scope":1,"blimp_passive_1":1,"blimp_passive_2":1,"blimp_principle_A_c_command":1,"blimp_principle_A_case_1":1,"blimp_principle_A_case_2":1,"blimp_principle_A_domain_1":1,"blimp_principle_A_domain_2":1,"blimp_principle_A_domain_3":1,"blimp_principle_A_reconstruction":1,"blimp_regular_plural_subject_verb_agreement_1":1,"blimp_regular_plural_subject_verb_agreement_2":1,"blimp_sentential_negation_npi_licensor_present":1,"blimp_sentential_negation_npi_scope":1,"blimp_sentential_subject_island":1,"blimp_superlative_quantifiers_1":1,"blimp_superlative_quantifiers_2":1,"blimp_tough_vs_raising_1":1,"blimp_tough_vs_raising_2":1,"blimp_transitive":1,"blimp_wh_island":1,"blimp_wh_questions_object_gap":1,"blimp_wh_questions_subject_gap":1,"blimp_wh_questions_subject_gap_long_distance":1,"blimp_wh_vs_that_no_gap":1,"blimp_wh_vs_that_no_gap_long_distance":1,"blimp_wh_vs_that_with_gap":1,"blimp_wh_vs_that_with_gap_long_distance":1,"commonsense_qa_text":"Yaml","hellaswag":1,"piqa":1}},"rank_range":[15,16],"release_date":"2026-06-19","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt-s2-5m/20260716T154647Z-v1.1","source":"AxiomicLabs/GPT-S2-5M","track":"base","updated_at":"2026-07-16T15:46:53.057636+00:00","vocab_size":4096},{"artifacts":{"result_file":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/models__nanowhale-100m-base/results_2026-07-20T01-41-02.427304.json","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z"},"benchmarks":{"arc_challenge":{"accuracy":0.22440273037542663,"chance":0.25,"ci":[-6.484641638225256,-0.22753128555176305],"classification":"below chance","label":"ARC-Challenge","n_samples":1172,"score":-3.4129692832764498},"arc_easy":{"accuracy":0.3728956228956229,"chance":0.25,"ci":[13.692480359147021,19.023569023569024],"classification":"reliably above chance","label":"ARC-Easy","n_samples":2376,"score":16.38608305274972},"blimp":{"accuracy":0.7091933531746032,"categories":{"Anaphor agreement":0.794,"Argument structure":0.7473333333333333,"Binding":0.7195714285714286,"Control / raising":0.6965999999999999,"Determiner-noun agreement":0.8567499999999999,"Ellipsis":0.7050000000000001,"Filler-gap":0.6524285714285715,"Irregular forms":0.917,"Island effects":0.495625,"NPI licensing":0.5134285714285715,"Quantifiers":0.64475,"Subject-verb agreement":0.7678333333333334},"category_count":12,"chance":0.5,"ci":[40.99927827380954,42.33725859788362],"classification":"reliably above chance","label":"BLiMP","n_samples":70000,"score":41.83867063492064},"commonsense_qa":{"accuracy":0.27764127764127766,"chance":0.2,"ci":[6.63390663390663,12.878787878787877],"classification":"reliably above chance","label":"CommonsenseQA","n_samples":1221,"score":9.705159705159705},"hellaswag":{"accuracy":0.27126070503883687,"chance":0.25,"ci":[1.63977959237868,3.976631481112662],"classification":"reliably above chance","label":"HellaSwag","n_samples":10042,"score":2.8347606718449163},"piqa":{"accuracy":0.573449401523395,"chance":0.5,"ci":[10.228509249183904,19.26006528835691],"classification":"reliably above chance","label":"PIQA","n_samples":1838,"score":14.689880304678994}},"complete":true,"composite":{"ci":[16.768369462355462,18.88831197015816],"classification":"reliably above chance","score":17.857047764529888},"id":"nanowhale-100m-base","name":"nanowhale 100M Base","params_m":110,"protocol":{"bootstrap_replicates":2000,"bootstrap_seed":20260716,"harness":"lm-eval==0.4.12","harness_git_hash":"dc3c1cf","inference_config":{"batch_size":"32","batch_sizes":[],"bootstrap_iters":100000,"device":"cuda:0","fewshot_seed":1234,"gen_kwargs":{},"limit":null,"model":"hf","model_args":{"dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true},"model_dtype":"torch.float32","model_num_parameters":110369621,"model_revision":"main","model_sha":"","numpy_seed":1234,"random_seed":1234,"torch_seed":1234,"use_cache":null},"n_shot":{"arc_challenge":0,"arc_easy":0,"blimp":0,"blimp_adjunct_island":0,"blimp_anaphor_gender_agreement":0,"blimp_anaphor_number_agreement":0,"blimp_animate_subject_passive":0,"blimp_animate_subject_trans":0,"blimp_causative":0,"blimp_complex_NP_island":0,"blimp_coordinate_structure_constraint_complex_left_branch":0,"blimp_coordinate_structure_constraint_object_extraction":0,"blimp_determiner_noun_agreement_1":0,"blimp_determiner_noun_agreement_2":0,"blimp_determiner_noun_agreement_irregular_1":0,"blimp_determiner_noun_agreement_irregular_2":0,"blimp_determiner_noun_agreement_with_adj_2":0,"blimp_determiner_noun_agreement_with_adj_irregular_1":0,"blimp_determiner_noun_agreement_with_adj_irregular_2":0,"blimp_determiner_noun_agreement_with_adjective_1":0,"blimp_distractor_agreement_relational_noun":0,"blimp_distractor_agreement_relative_clause":0,"blimp_drop_argument":0,"blimp_ellipsis_n_bar_1":0,"blimp_ellipsis_n_bar_2":0,"blimp_existential_there_object_raising":0,"blimp_existential_there_quantifiers_1":0,"blimp_existential_there_quantifiers_2":0,"blimp_existential_there_subject_raising":0,"blimp_expletive_it_object_raising":0,"blimp_inchoative":0,"blimp_intransitive":0,"blimp_irregular_past_participle_adjectives":0,"blimp_irregular_past_participle_verbs":0,"blimp_irregular_plural_subject_verb_agreement_1":0,"blimp_irregular_plural_subject_verb_agreement_2":0,"blimp_left_branch_island_echo_question":0,"blimp_left_branch_island_simple_question":0,"blimp_matrix_question_npi_licensor_present":0,"blimp_npi_present_1":0,"blimp_npi_present_2":0,"blimp_only_npi_licensor_present":0,"blimp_only_npi_scope":0,"blimp_passive_1":0,"blimp_passive_2":0,"blimp_principle_A_c_command":0,"blimp_principle_A_case_1":0,"blimp_principle_A_case_2":0,"blimp_principle_A_domain_1":0,"blimp_principle_A_domain_2":0,"blimp_principle_A_domain_3":0,"blimp_principle_A_reconstruction":0,"blimp_regular_plural_subject_verb_agreement_1":0,"blimp_regular_plural_subject_verb_agreement_2":0,"blimp_sentential_negation_npi_licensor_present":0,"blimp_sentential_negation_npi_scope":0,"blimp_sentential_subject_island":0,"blimp_superlative_quantifiers_1":0,"blimp_superlative_quantifiers_2":0,"blimp_tough_vs_raising_1":0,"blimp_tough_vs_raising_2":0,"blimp_transitive":0,"blimp_wh_island":0,"blimp_wh_questions_object_gap":0,"blimp_wh_questions_subject_gap":0,"blimp_wh_questions_subject_gap_long_distance":0,"blimp_wh_vs_that_no_gap":0,"blimp_wh_vs_that_no_gap_long_distance":0,"blimp_wh_vs_that_with_gap":0,"blimp_wh_vs_that_with_gap_long_distance":0,"commonsense_qa_text":0,"hellaswag":0,"piqa":0},"suite":"slm-benchmark-v1.1","task_config":{"arc_challenge":{"dataset_name":"ARC-Challenge","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_challenge.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_challenge","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"arc_easy":{"dataset_name":"ARC-Easy","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_easy.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_easy","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"blimp_adjunct_island":{"dataset_name":"adjunct_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/adjunct_island.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_adjunct_island","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_gender_agreement":{"dataset_name":"anaphor_gender_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_gender_agreement","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_number_agreement":{"dataset_name":"anaphor_number_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_number_agreement.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_number_agreement","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_passive":{"dataset_name":"animate_subject_passive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_passive.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_passive","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_trans":{"dataset_name":"animate_subject_trans","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_trans.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_trans","unsafe_code":false,"validation_split":"train"},"blimp_causative":{"dataset_name":"causative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/causative.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_causative","unsafe_code":false,"validation_split":"train"},"blimp_complex_NP_island":{"dataset_name":"complex_NP_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/complex_NP_island.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_complex_NP_island","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_complex_left_branch":{"dataset_name":"coordinate_structure_constraint_complex_left_branch","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_complex_left_branch","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_object_extraction":{"dataset_name":"coordinate_structure_constraint_object_extraction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_object_extraction","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_1":{"dataset_name":"determiner_noun_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_2":{"dataset_name":"determiner_noun_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_1":{"dataset_name":"determiner_noun_agreement_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_2":{"dataset_name":"determiner_noun_agreement_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_2":{"dataset_name":"determiner_noun_agreement_with_adj_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_1":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_2":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adjective_1":{"dataset_name":"determiner_noun_agreement_with_adjective_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adjective_1","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relational_noun":{"dataset_name":"distractor_agreement_relational_noun","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relational_noun","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relative_clause":{"dataset_name":"distractor_agreement_relative_clause","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relative_clause","unsafe_code":false,"validation_split":"train"},"blimp_drop_argument":{"dataset_name":"drop_argument","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/drop_argument.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_drop_argument","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_1":{"dataset_name":"ellipsis_n_bar_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_1","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_2":{"dataset_name":"ellipsis_n_bar_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_object_raising":{"dataset_name":"existential_there_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_object_raising.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_1":{"dataset_name":"existential_there_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_2":{"dataset_name":"existential_there_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_subject_raising":{"dataset_name":"existential_there_subject_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_subject_raising.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_subject_raising","unsafe_code":false,"validation_split":"train"},"blimp_expletive_it_object_raising":{"dataset_name":"expletive_it_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/expletive_it_object_raising.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_expletive_it_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_inchoative":{"dataset_name":"inchoative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/inchoative.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_inchoative","unsafe_code":false,"validation_split":"train"},"blimp_intransitive":{"dataset_name":"intransitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/intransitive.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_intransitive","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_adjectives":{"dataset_name":"irregular_past_participle_adjectives","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_adjectives","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_verbs":{"dataset_name":"irregular_past_participle_verbs","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_verbs","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_1":{"dataset_name":"irregular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_2":{"dataset_name":"irregular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_echo_question":{"dataset_name":"left_branch_island_echo_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_echo_question","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_simple_question":{"dataset_name":"left_branch_island_simple_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_simple_question","unsafe_code":false,"validation_split":"train"},"blimp_matrix_question_npi_licensor_present":{"dataset_name":"matrix_question_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_matrix_question_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_1":{"dataset_name":"npi_present_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_1.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_1","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_2":{"dataset_name":"npi_present_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_2.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_2","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_licensor_present":{"dataset_name":"only_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_licensor_present.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_scope":{"dataset_name":"only_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_scope.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_passive_1":{"dataset_name":"passive_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_1.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_1","unsafe_code":false,"validation_split":"train"},"blimp_passive_2":{"dataset_name":"passive_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_2.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_c_command":{"dataset_name":"principle_A_c_command","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_c_command.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_c_command","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_1":{"dataset_name":"principle_A_case_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_1.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_2":{"dataset_name":"principle_A_case_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_2.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_1":{"dataset_name":"principle_A_domain_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_1.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_2":{"dataset_name":"principle_A_domain_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_2.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_3":{"dataset_name":"principle_A_domain_3","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_3.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_3","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_reconstruction":{"dataset_name":"principle_A_reconstruction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_reconstruction.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_reconstruction","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_1":{"dataset_name":"regular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_2":{"dataset_name":"regular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_licensor_present":{"dataset_name":"sentential_negation_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_scope":{"dataset_name":"sentential_negation_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_sentential_subject_island":{"dataset_name":"sentential_subject_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_subject_island.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_subject_island","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_1":{"dataset_name":"superlative_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_2":{"dataset_name":"superlative_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_1":{"dataset_name":"tough_vs_raising_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_1.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_1","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_2":{"dataset_name":"tough_vs_raising_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_2.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_2","unsafe_code":false,"validation_split":"train"},"blimp_transitive":{"dataset_name":"transitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/transitive.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_transitive","unsafe_code":false,"validation_split":"train"},"blimp_wh_island":{"dataset_name":"wh_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_island.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_island","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_object_gap":{"dataset_name":"wh_questions_object_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_object_gap.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_object_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap":{"dataset_name":"wh_questions_subject_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap_long_distance":{"dataset_name":"wh_questions_subject_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap":{"dataset_name":"wh_vs_that_no_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap_long_distance":{"dataset_name":"wh_vs_that_no_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap":{"dataset_name":"wh_vs_that_with_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap_long_distance":{"dataset_name":"wh_vs_that_with_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap_long_distance","unsafe_code":false,"validation_split":"train"},"commonsense_qa_text":{"dataset_path":"tau/commonsense_qa","description":"","doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/config/tasks/commonsense_qa_text.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"commonsense_qa_text","training_split":"train","unsafe_code":false,"validation_split":"validation"},"hellaswag":{"dataset_path":"Rowan/hellaswag","description":"","doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_config":{"doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":"<function process_docs at 0x75209af931a0>","sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","process_docs":"def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\\n    def _process_doc(doc):\\n        ctx = doc[\\"ctx_a\\"] + \\" \\" + doc[\\"ctx_b\\"].capitalize()\\n        out_doc = {\\n            \\"query\\": preprocess(doc[\\"activity_label\\"] + \\": \\" + ctx),\\n            \\"choices\\": [preprocess(ending) for ending in doc[\\"endings\\"]],\\n            \\"gold\\": int(doc[\\"label\\"]),\\n        }\\n        return out_doc\\n\\n    return dataset.map(_process_doc)\\n","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"hellaswag","training_split":"train","unsafe_code":false,"validation_split":"validation"},"piqa":{"dataset_path":"baber/piqa","description":"","doc_to_choice":"{{[sol1, sol2]}}","doc_to_decontamination_query":"goal","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{[sol1, sol2]}}","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/piqa/piqa.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"piqa","training_split":"train","unsafe_code":false,"validation_split":"validation"}},"task_versions":{"arc_challenge":1,"arc_easy":1,"blimp":"2.0","blimp_adjunct_island":1,"blimp_anaphor_gender_agreement":1,"blimp_anaphor_number_agreement":1,"blimp_animate_subject_passive":1,"blimp_animate_subject_trans":1,"blimp_causative":1,"blimp_complex_NP_island":1,"blimp_coordinate_structure_constraint_complex_left_branch":1,"blimp_coordinate_structure_constraint_object_extraction":1,"blimp_determiner_noun_agreement_1":1,"blimp_determiner_noun_agreement_2":1,"blimp_determiner_noun_agreement_irregular_1":1,"blimp_determiner_noun_agreement_irregular_2":1,"blimp_determiner_noun_agreement_with_adj_2":1,"blimp_determiner_noun_agreement_with_adj_irregular_1":1,"blimp_determiner_noun_agreement_with_adj_irregular_2":1,"blimp_determiner_noun_agreement_with_adjective_1":1,"blimp_distractor_agreement_relational_noun":1,"blimp_distractor_agreement_relative_clause":1,"blimp_drop_argument":1,"blimp_ellipsis_n_bar_1":1,"blimp_ellipsis_n_bar_2":1,"blimp_existential_there_object_raising":1,"blimp_existential_there_quantifiers_1":1,"blimp_existential_there_quantifiers_2":1,"blimp_existential_there_subject_raising":1,"blimp_expletive_it_object_raising":1,"blimp_inchoative":1,"blimp_intransitive":1,"blimp_irregular_past_participle_adjectives":1,"blimp_irregular_past_participle_verbs":1,"blimp_irregular_plural_subject_verb_agreement_1":1,"blimp_irregular_plural_subject_verb_agreement_2":1,"blimp_left_branch_island_echo_question":1,"blimp_left_branch_island_simple_question":1,"blimp_matrix_question_npi_licensor_present":1,"blimp_npi_present_1":1,"blimp_npi_present_2":1,"blimp_only_npi_licensor_present":1,"blimp_only_npi_scope":1,"blimp_passive_1":1,"blimp_passive_2":1,"blimp_principle_A_c_command":1,"blimp_principle_A_case_1":1,"blimp_principle_A_case_2":1,"blimp_principle_A_domain_1":1,"blimp_principle_A_domain_2":1,"blimp_principle_A_domain_3":1,"blimp_principle_A_reconstruction":1,"blimp_regular_plural_subject_verb_agreement_1":1,"blimp_regular_plural_subject_verb_agreement_2":1,"blimp_sentential_negation_npi_licensor_present":1,"blimp_sentential_negation_npi_scope":1,"blimp_sentential_subject_island":1,"blimp_superlative_quantifiers_1":1,"blimp_superlative_quantifiers_2":1,"blimp_tough_vs_raising_1":1,"blimp_tough_vs_raising_2":1,"blimp_transitive":1,"blimp_wh_island":1,"blimp_wh_questions_object_gap":1,"blimp_wh_questions_subject_gap":1,"blimp_wh_questions_subject_gap_long_distance":1,"blimp_wh_vs_that_no_gap":1,"blimp_wh_vs_that_no_gap_long_distance":1,"blimp_wh_vs_that_with_gap":1,"blimp_wh_vs_that_with_gap_long_distance":1,"commonsense_qa_text":"Yaml","hellaswag":1,"piqa":1}},"rank_range":[17,18],"release_date":"2026-04-24","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z","source":"HuggingFaceTB/nanowhale-100m-base","track":"base","updated_at":"2026-07-19T23:41:08.863409+00:00","vocab_size":129280},{"artifacts":{"result_file":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/pythia-31m/20260716T154647Z-v1.1/results_v1.1_merged.json","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/pythia-31m/20260716T154647Z-v1.1"},"benchmarks":{"arc_challenge":{"accuracy":0.2167235494880546,"chance":0.25,"ci":[-7.622298065984071,-1.1376564277588153],"classification":"below chance","label":"ARC-Challenge","n_samples":1172,"score":-4.436860068259385},"arc_easy":{"accuracy":0.3409090909090909,"chance":0.25,"ci":[9.652076318742987,14.534231200897866],"classification":"reliably above chance","label":"ARC-Easy","n_samples":2376,"score":12.121212121212118},"blimp":{"accuracy":0.7167433531746031,"categories":{"Anaphor agreement":0.9285,"Argument structure":0.7563333333333333,"Binding":0.7057142857142856,"Control / raising":0.7332,"Determiner-noun agreement":0.857375,"Ellipsis":0.6745,"Filler-gap":0.6648571428571428,"Irregular forms":0.8815,"Island effects":0.535,"NPI licensing":0.45985714285714285,"Quantifiers":0.62375,"Subject-verb agreement":0.7803333333333334},"category_count":12,"chance":0.5,"ci":[42.51403703703704,43.85099421296294],"classification":"reliably above chance","label":"BLiMP","n_samples":70000,"score":43.34867063492063},"commonsense_qa":{"accuracy":0.26453726453726456,"chance":0.2,"ci":[5.098280098280095,11.138411138411136],"classification":"reliably above chance","label":"CommonsenseQA","n_samples":1221,"score":8.067158067158068},"hellaswag":{"accuracy":0.27185819557857,"chance":0.25,"ci":[1.7725552678749275,4.069906393148773],"classification":"reliably above chance","label":"HellaSwag","n_samples":10042,"score":2.9144260771426658},"piqa":{"accuracy":0.5680087051142546,"chance":0.5,"ci":[9.140369967355833,18.063112078346034],"classification":"reliably above chance","label":"PIQA","n_samples":1838,"score":13.601741022850922}},"complete":true,"composite":{"ci":[15.828302485043055,17.965860323148288],"classification":"reliably above chance","score":16.92981203838316},"id":"pythia-31m","name":"Pythia 31M","params_m":31,"protocol":{"bootstrap_replicates":2000,"bootstrap_seed":20260716,"harness":"lm-eval==0.4.12","harness_git_hash":null,"inference_config":{"batch_size":"auto","batch_sizes":[64],"bootstrap_iters":100000,"device":"cuda:0","fewshot_seed":1234,"gen_kwargs":{},"limit":null,"model":"hf","model_args":{"dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false},"model_dtype":"torch.bfloat16","model_num_parameters":30494720,"model_revision":"main","model_sha":"","numpy_seed":1234,"random_seed":1234,"tasks":["hellaswag","piqa","arc_easy","arc_challenge","commonsense_qa_text","blimp"],"torch_seed":1234,"use_cache":null,"v1_1_sources":{"rechecked_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/pythia-31m/20260716T154341Z-v1.1/results_v1.1_merged.json","unchanged_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/pythia-31m/20260716T154341Z-v1.1/results_v1.1_merged.json"}},"n_shot":{"arc_easy":0,"blimp":0,"blimp_adjunct_island":0,"blimp_anaphor_gender_agreement":0,"blimp_anaphor_number_agreement":0,"blimp_animate_subject_passive":0,"blimp_animate_subject_trans":0,"blimp_causative":0,"blimp_complex_NP_island":0,"blimp_coordinate_structure_constraint_complex_left_branch":0,"blimp_coordinate_structure_constraint_object_extraction":0,"blimp_determiner_noun_agreement_1":0,"blimp_determiner_noun_agreement_2":0,"blimp_determiner_noun_agreement_irregular_1":0,"blimp_determiner_noun_agreement_irregular_2":0,"blimp_determiner_noun_agreement_with_adj_2":0,"blimp_determiner_noun_agreement_with_adj_irregular_1":0,"blimp_determiner_noun_agreement_with_adj_irregular_2":0,"blimp_determiner_noun_agreement_with_adjective_1":0,"blimp_distractor_agreement_relational_noun":0,"blimp_distractor_agreement_relative_clause":0,"blimp_drop_argument":0,"blimp_ellipsis_n_bar_1":0,"blimp_ellipsis_n_bar_2":0,"blimp_existential_there_object_raising":0,"blimp_existential_there_quantifiers_1":0,"blimp_existential_there_quantifiers_2":0,"blimp_existential_there_subject_raising":0,"blimp_expletive_it_object_raising":0,"blimp_inchoative":0,"blimp_intransitive":0,"blimp_irregular_past_participle_adjectives":0,"blimp_irregular_past_participle_verbs":0,"blimp_irregular_plural_subject_verb_agreement_1":0,"blimp_irregular_plural_subject_verb_agreement_2":0,"blimp_left_branch_island_echo_question":0,"blimp_left_branch_island_simple_question":0,"blimp_matrix_question_npi_licensor_present":0,"blimp_npi_present_1":0,"blimp_npi_present_2":0,"blimp_only_npi_licensor_present":0,"blimp_only_npi_scope":0,"blimp_passive_1":0,"blimp_passive_2":0,"blimp_principle_A_c_command":0,"blimp_principle_A_case_1":0,"blimp_principle_A_case_2":0,"blimp_principle_A_domain_1":0,"blimp_principle_A_domain_2":0,"blimp_principle_A_domain_3":0,"blimp_principle_A_reconstruction":0,"blimp_regular_plural_subject_verb_agreement_1":0,"blimp_regular_plural_subject_verb_agreement_2":0,"blimp_sentential_negation_npi_licensor_present":0,"blimp_sentential_negation_npi_scope":0,"blimp_sentential_subject_island":0,"blimp_superlative_quantifiers_1":0,"blimp_superlative_quantifiers_2":0,"blimp_tough_vs_raising_1":0,"blimp_tough_vs_raising_2":0,"blimp_transitive":0,"blimp_wh_island":0,"blimp_wh_questions_object_gap":0,"blimp_wh_questions_subject_gap":0,"blimp_wh_questions_subject_gap_long_distance":0,"blimp_wh_vs_that_no_gap":0,"blimp_wh_vs_that_no_gap_long_distance":0,"blimp_wh_vs_that_with_gap":0,"blimp_wh_vs_that_with_gap_long_distance":0,"commonsense_qa":0,"hellaswag":0,"piqa":0},"suite":"slm-benchmark-v1.1","task_config":{"arc_challenge":{"dataset_name":"ARC-Challenge","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_challenge.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_challenge","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"arc_easy":{"dataset_name":"ARC-Easy","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_easy.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_easy","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"blimp_adjunct_island":{"dataset_name":"adjunct_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/adjunct_island.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_adjunct_island","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_gender_agreement":{"dataset_name":"anaphor_gender_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_gender_agreement","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_number_agreement":{"dataset_name":"anaphor_number_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_number_agreement.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_number_agreement","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_passive":{"dataset_name":"animate_subject_passive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_passive.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_passive","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_trans":{"dataset_name":"animate_subject_trans","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_trans.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_trans","unsafe_code":false,"validation_split":"train"},"blimp_causative":{"dataset_name":"causative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/causative.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_causative","unsafe_code":false,"validation_split":"train"},"blimp_complex_NP_island":{"dataset_name":"complex_NP_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/complex_NP_island.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_complex_NP_island","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_complex_left_branch":{"dataset_name":"coordinate_structure_constraint_complex_left_branch","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_complex_left_branch","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_object_extraction":{"dataset_name":"coordinate_structure_constraint_object_extraction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_object_extraction","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_1":{"dataset_name":"determiner_noun_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_2":{"dataset_name":"determiner_noun_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_1":{"dataset_name":"determiner_noun_agreement_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_2":{"dataset_name":"determiner_noun_agreement_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_2":{"dataset_name":"determiner_noun_agreement_with_adj_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_1":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_2":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adjective_1":{"dataset_name":"determiner_noun_agreement_with_adjective_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adjective_1","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relational_noun":{"dataset_name":"distractor_agreement_relational_noun","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relational_noun","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relative_clause":{"dataset_name":"distractor_agreement_relative_clause","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relative_clause","unsafe_code":false,"validation_split":"train"},"blimp_drop_argument":{"dataset_name":"drop_argument","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/drop_argument.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_drop_argument","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_1":{"dataset_name":"ellipsis_n_bar_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_1","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_2":{"dataset_name":"ellipsis_n_bar_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_object_raising":{"dataset_name":"existential_there_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_object_raising.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_1":{"dataset_name":"existential_there_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_2":{"dataset_name":"existential_there_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_subject_raising":{"dataset_name":"existential_there_subject_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_subject_raising.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_subject_raising","unsafe_code":false,"validation_split":"train"},"blimp_expletive_it_object_raising":{"dataset_name":"expletive_it_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/expletive_it_object_raising.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_expletive_it_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_inchoative":{"dataset_name":"inchoative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/inchoative.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_inchoative","unsafe_code":false,"validation_split":"train"},"blimp_intransitive":{"dataset_name":"intransitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/intransitive.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_intransitive","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_adjectives":{"dataset_name":"irregular_past_participle_adjectives","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_adjectives","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_verbs":{"dataset_name":"irregular_past_participle_verbs","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_verbs","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_1":{"dataset_name":"irregular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_2":{"dataset_name":"irregular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_echo_question":{"dataset_name":"left_branch_island_echo_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_echo_question","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_simple_question":{"dataset_name":"left_branch_island_simple_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_simple_question","unsafe_code":false,"validation_split":"train"},"blimp_matrix_question_npi_licensor_present":{"dataset_name":"matrix_question_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_matrix_question_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_1":{"dataset_name":"npi_present_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_1","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_2":{"dataset_name":"npi_present_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_2","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_licensor_present":{"dataset_name":"only_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_scope":{"dataset_name":"only_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_passive_1":{"dataset_name":"passive_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_1","unsafe_code":false,"validation_split":"train"},"blimp_passive_2":{"dataset_name":"passive_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_c_command":{"dataset_name":"principle_A_c_command","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_c_command.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_c_command","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_1":{"dataset_name":"principle_A_case_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_2":{"dataset_name":"principle_A_case_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_1":{"dataset_name":"principle_A_domain_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_2":{"dataset_name":"principle_A_domain_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_3":{"dataset_name":"principle_A_domain_3","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_3.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_3","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_reconstruction":{"dataset_name":"principle_A_reconstruction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_reconstruction.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_reconstruction","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_1":{"dataset_name":"regular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_2":{"dataset_name":"regular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_licensor_present":{"dataset_name":"sentential_negation_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_scope":{"dataset_name":"sentential_negation_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_sentential_subject_island":{"dataset_name":"sentential_subject_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_subject_island.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_subject_island","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_1":{"dataset_name":"superlative_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_2":{"dataset_name":"superlative_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_1":{"dataset_name":"tough_vs_raising_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_1","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_2":{"dataset_name":"tough_vs_raising_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_2","unsafe_code":false,"validation_split":"train"},"blimp_transitive":{"dataset_name":"transitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/transitive.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_transitive","unsafe_code":false,"validation_split":"train"},"blimp_wh_island":{"dataset_name":"wh_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_island.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_island","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_object_gap":{"dataset_name":"wh_questions_object_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_object_gap.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_object_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap":{"dataset_name":"wh_questions_subject_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap_long_distance":{"dataset_name":"wh_questions_subject_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap":{"dataset_name":"wh_vs_that_no_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap_long_distance":{"dataset_name":"wh_vs_that_no_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap":{"dataset_name":"wh_vs_that_with_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap_long_distance":{"dataset_name":"wh_vs_that_with_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap_long_distance","unsafe_code":false,"validation_split":"train"},"commonsense_qa_text":{"dataset_path":"tau/commonsense_qa","description":"","doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/config/tasks/commonsense_qa_text.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"commonsense_qa_text","training_split":"train","unsafe_code":false,"validation_split":"validation"},"hellaswag":{"dataset_path":"Rowan/hellaswag","description":"","doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_config":{"doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":"<function process_docs at 0x7ce4241c0180>","sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","process_docs":"def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\\n    def _process_doc(doc):\\n        ctx = doc[\\"ctx_a\\"] + \\" \\" + doc[\\"ctx_b\\"].capitalize()\\n        out_doc = {\\n            \\"query\\": preprocess(doc[\\"activity_label\\"] + \\": \\" + ctx),\\n            \\"choices\\": [preprocess(ending) for ending in doc[\\"endings\\"]],\\n            \\"gold\\": int(doc[\\"label\\"]),\\n        }\\n        return out_doc\\n\\n    return dataset.map(_process_doc)\\n","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"hellaswag","training_split":"train","unsafe_code":false,"validation_split":"validation"},"piqa":{"dataset_path":"baber/piqa","description":"","doc_to_choice":"{{[sol1, sol2]}}","doc_to_decontamination_query":"goal","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{[sol1, sol2]}}","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/piqa/piqa.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"piqa","training_split":"train","unsafe_code":false,"validation_split":"validation"}},"task_versions":{"arc_challenge":1,"arc_easy":1,"blimp":"2.0","blimp_adjunct_island":1,"blimp_anaphor_gender_agreement":1,"blimp_anaphor_number_agreement":1,"blimp_animate_subject_passive":1,"blimp_animate_subject_trans":1,"blimp_causative":1,"blimp_complex_NP_island":1,"blimp_coordinate_structure_constraint_complex_left_branch":1,"blimp_coordinate_structure_constraint_object_extraction":1,"blimp_determiner_noun_agreement_1":1,"blimp_determiner_noun_agreement_2":1,"blimp_determiner_noun_agreement_irregular_1":1,"blimp_determiner_noun_agreement_irregular_2":1,"blimp_determiner_noun_agreement_with_adj_2":1,"blimp_determiner_noun_agreement_with_adj_irregular_1":1,"blimp_determiner_noun_agreement_with_adj_irregular_2":1,"blimp_determiner_noun_agreement_with_adjective_1":1,"blimp_distractor_agreement_relational_noun":1,"blimp_distractor_agreement_relative_clause":1,"blimp_drop_argument":1,"blimp_ellipsis_n_bar_1":1,"blimp_ellipsis_n_bar_2":1,"blimp_existential_there_object_raising":1,"blimp_existential_there_quantifiers_1":1,"blimp_existential_there_quantifiers_2":1,"blimp_existential_there_subject_raising":1,"blimp_expletive_it_object_raising":1,"blimp_inchoative":1,"blimp_intransitive":1,"blimp_irregular_past_participle_adjectives":1,"blimp_irregular_past_participle_verbs":1,"blimp_irregular_plural_subject_verb_agreement_1":1,"blimp_irregular_plural_subject_verb_agreement_2":1,"blimp_left_branch_island_echo_question":1,"blimp_left_branch_island_simple_question":1,"blimp_matrix_question_npi_licensor_present":1,"blimp_npi_present_1":1,"blimp_npi_present_2":1,"blimp_only_npi_licensor_present":1,"blimp_only_npi_scope":1,"blimp_passive_1":1,"blimp_passive_2":1,"blimp_principle_A_c_command":1,"blimp_principle_A_case_1":1,"blimp_principle_A_case_2":1,"blimp_principle_A_domain_1":1,"blimp_principle_A_domain_2":1,"blimp_principle_A_domain_3":1,"blimp_principle_A_reconstruction":1,"blimp_regular_plural_subject_verb_agreement_1":1,"blimp_regular_plural_subject_verb_agreement_2":1,"blimp_sentential_negation_npi_licensor_present":1,"blimp_sentential_negation_npi_scope":1,"blimp_sentential_subject_island":1,"blimp_superlative_quantifiers_1":1,"blimp_superlative_quantifiers_2":1,"blimp_tough_vs_raising_1":1,"blimp_tough_vs_raising_2":1,"blimp_transitive":1,"blimp_wh_island":1,"blimp_wh_questions_object_gap":1,"blimp_wh_questions_subject_gap":1,"blimp_wh_questions_subject_gap_long_distance":1,"blimp_wh_vs_that_no_gap":1,"blimp_wh_vs_that_no_gap_long_distance":1,"blimp_wh_vs_that_with_gap":1,"blimp_wh_vs_that_with_gap_long_distance":1,"commonsense_qa_text":"Yaml","hellaswag":1,"piqa":1}},"rank_range":[17,18],"release_date":"2026-02-24","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/pythia-31m/20260716T154647Z-v1.1","source":"EleutherAI/pythia-31m","track":"base","updated_at":"2026-07-16T15:47:00.508307+00:00","vocab_size":50304},{"artifacts":{"result_file":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/atom-3-4m/20260716T154647Z-v1.1/results_v1.1_merged.json","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/atom-3-4m/20260716T154647Z-v1.1"},"benchmarks":{"arc_challenge":{"accuracy":0.21843003412969283,"chance":0.25,"ci":[-7.394766780432309,-1.1376564277588153],"classification":"below chance","label":"ARC-Challenge","n_samples":1172,"score":-4.209328782707622},"arc_easy":{"accuracy":0.33080808080808083,"chance":0.25,"ci":[8.361391694725029,13.24354657687991],"classification":"reliably above chance","label":"ARC-Easy","n_samples":2376,"score":10.774410774410777},"blimp":{"accuracy":0.6819768518518519,"categories":{"Anaphor agreement":0.6659999999999999,"Argument structure":0.7285555555555556,"Binding":0.713857142857143,"Control / raising":0.6869999999999999,"Determiner-noun agreement":0.842125,"Ellipsis":0.708,"Filler-gap":0.6594285714285714,"Irregular forms":0.845,"Island effects":0.5431250000000001,"NPI licensing":0.5187142857142858,"Quantifiers":0.53975,"Subject-verb agreement":0.7321666666666666},"category_count":12,"chance":0.5,"ci":[35.579373181216944,36.99308052248677],"classification":"reliably above chance","label":"BLiMP","n_samples":70000,"score":36.39537037037037},"commonsense_qa":{"accuracy":0.24733824733824733,"chance":0.2,"ci":[2.9484029484029453,9.193284193284189],"classification":"reliably above chance","label":"CommonsenseQA","n_samples":1221,"score":5.917280917280914},"hellaswag":{"accuracy":0.27614021111332404,"chance":0.25,"ci":[2.356768240058423,4.547566885746535],"classification":"reliably above chance","label":"HellaSwag","n_samples":10042,"score":3.4853614817765384},"piqa":{"accuracy":0.5571273122959739,"chance":0.5,"ci":[6.855277475516863,15.88683351468989],"classification":"reliably above chance","label":"PIQA","n_samples":1838,"score":11.42546245919478}},"complete":true,"composite":{"ci":[13.263758832507774,15.316045445100848],"classification":"reliably above chance","score":14.327312373584093},"id":"atom-3-4m","name":"Atom 3.4M","params_m":3.4128,"protocol":{"bootstrap_replicates":2000,"bootstrap_seed":20260716,"harness":"lm-eval==0.4.12","harness_git_hash":null,"inference_config":{"batch_size":"auto","batch_sizes":[64],"bootstrap_iters":100000,"device":"cuda:0","fewshot_seed":1234,"gen_kwargs":{},"limit":null,"model":"hf","model_args":{"dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true},"model_dtype":"torch.float32","model_num_parameters":3412800,"model_revision":"main","model_sha":"","numpy_seed":1234,"random_seed":1234,"tasks":["hellaswag","piqa","arc_easy","arc_challenge","commonsense_qa_text","blimp"],"torch_seed":1234,"use_cache":null,"v1_1_sources":{"rechecked_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/atom-3-4m/20260716T154341Z-v1.1/results_v1.1_merged.json","unchanged_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/atom-3-4m/20260716T154341Z-v1.1/results_v1.1_merged.json"}},"n_shot":{"arc_easy":0,"blimp":0,"blimp_adjunct_island":0,"blimp_anaphor_gender_agreement":0,"blimp_anaphor_number_agreement":0,"blimp_animate_subject_passive":0,"blimp_animate_subject_trans":0,"blimp_causative":0,"blimp_complex_NP_island":0,"blimp_coordinate_structure_constraint_complex_left_branch":0,"blimp_coordinate_structure_constraint_object_extraction":0,"blimp_determiner_noun_agreement_1":0,"blimp_determiner_noun_agreement_2":0,"blimp_determiner_noun_agreement_irregular_1":0,"blimp_determiner_noun_agreement_irregular_2":0,"blimp_determiner_noun_agreement_with_adj_2":0,"blimp_determiner_noun_agreement_with_adj_irregular_1":0,"blimp_determiner_noun_agreement_with_adj_irregular_2":0,"blimp_determiner_noun_agreement_with_adjective_1":0,"blimp_distractor_agreement_relational_noun":0,"blimp_distractor_agreement_relative_clause":0,"blimp_drop_argument":0,"blimp_ellipsis_n_bar_1":0,"blimp_ellipsis_n_bar_2":0,"blimp_existential_there_object_raising":0,"blimp_existential_there_quantifiers_1":0,"blimp_existential_there_quantifiers_2":0,"blimp_existential_there_subject_raising":0,"blimp_expletive_it_object_raising":0,"blimp_inchoative":0,"blimp_intransitive":0,"blimp_irregular_past_participle_adjectives":0,"blimp_irregular_past_participle_verbs":0,"blimp_irregular_plural_subject_verb_agreement_1":0,"blimp_irregular_plural_subject_verb_agreement_2":0,"blimp_left_branch_island_echo_question":0,"blimp_left_branch_island_simple_question":0,"blimp_matrix_question_npi_licensor_present":0,"blimp_npi_present_1":0,"blimp_npi_present_2":0,"blimp_only_npi_licensor_present":0,"blimp_only_npi_scope":0,"blimp_passive_1":0,"blimp_passive_2":0,"blimp_principle_A_c_command":0,"blimp_principle_A_case_1":0,"blimp_principle_A_case_2":0,"blimp_principle_A_domain_1":0,"blimp_principle_A_domain_2":0,"blimp_principle_A_domain_3":0,"blimp_principle_A_reconstruction":0,"blimp_regular_plural_subject_verb_agreement_1":0,"blimp_regular_plural_subject_verb_agreement_2":0,"blimp_sentential_negation_npi_licensor_present":0,"blimp_sentential_negation_npi_scope":0,"blimp_sentential_subject_island":0,"blimp_superlative_quantifiers_1":0,"blimp_superlative_quantifiers_2":0,"blimp_tough_vs_raising_1":0,"blimp_tough_vs_raising_2":0,"blimp_transitive":0,"blimp_wh_island":0,"blimp_wh_questions_object_gap":0,"blimp_wh_questions_subject_gap":0,"blimp_wh_questions_subject_gap_long_distance":0,"blimp_wh_vs_that_no_gap":0,"blimp_wh_vs_that_no_gap_long_distance":0,"blimp_wh_vs_that_with_gap":0,"blimp_wh_vs_that_with_gap_long_distance":0,"commonsense_qa":0,"hellaswag":0,"piqa":0},"suite":"slm-benchmark-v1.1","task_config":{"arc_challenge":{"dataset_name":"ARC-Challenge","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_challenge.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_challenge","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"arc_easy":{"dataset_name":"ARC-Easy","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_easy.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_easy","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"blimp_adjunct_island":{"dataset_name":"adjunct_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/adjunct_island.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_adjunct_island","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_gender_agreement":{"dataset_name":"anaphor_gender_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_gender_agreement","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_number_agreement":{"dataset_name":"anaphor_number_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_number_agreement.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_number_agreement","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_passive":{"dataset_name":"animate_subject_passive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_passive.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_passive","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_trans":{"dataset_name":"animate_subject_trans","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_trans.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_trans","unsafe_code":false,"validation_split":"train"},"blimp_causative":{"dataset_name":"causative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/causative.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_causative","unsafe_code":false,"validation_split":"train"},"blimp_complex_NP_island":{"dataset_name":"complex_NP_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/complex_NP_island.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_complex_NP_island","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_complex_left_branch":{"dataset_name":"coordinate_structure_constraint_complex_left_branch","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_complex_left_branch","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_object_extraction":{"dataset_name":"coordinate_structure_constraint_object_extraction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_object_extraction","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_1":{"dataset_name":"determiner_noun_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_2":{"dataset_name":"determiner_noun_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_1":{"dataset_name":"determiner_noun_agreement_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_2":{"dataset_name":"determiner_noun_agreement_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_2":{"dataset_name":"determiner_noun_agreement_with_adj_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_1":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_2":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adjective_1":{"dataset_name":"determiner_noun_agreement_with_adjective_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adjective_1","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relational_noun":{"dataset_name":"distractor_agreement_relational_noun","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relational_noun","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relative_clause":{"dataset_name":"distractor_agreement_relative_clause","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relative_clause","unsafe_code":false,"validation_split":"train"},"blimp_drop_argument":{"dataset_name":"drop_argument","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/drop_argument.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_drop_argument","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_1":{"dataset_name":"ellipsis_n_bar_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_1","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_2":{"dataset_name":"ellipsis_n_bar_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_object_raising":{"dataset_name":"existential_there_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_object_raising.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_1":{"dataset_name":"existential_there_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_2":{"dataset_name":"existential_there_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_subject_raising":{"dataset_name":"existential_there_subject_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_subject_raising.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_subject_raising","unsafe_code":false,"validation_split":"train"},"blimp_expletive_it_object_raising":{"dataset_name":"expletive_it_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/expletive_it_object_raising.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_expletive_it_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_inchoative":{"dataset_name":"inchoative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/inchoative.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_inchoative","unsafe_code":false,"validation_split":"train"},"blimp_intransitive":{"dataset_name":"intransitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/intransitive.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_intransitive","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_adjectives":{"dataset_name":"irregular_past_participle_adjectives","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_adjectives","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_verbs":{"dataset_name":"irregular_past_participle_verbs","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_verbs","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_1":{"dataset_name":"irregular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_2":{"dataset_name":"irregular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_echo_question":{"dataset_name":"left_branch_island_echo_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_echo_question","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_simple_question":{"dataset_name":"left_branch_island_simple_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_simple_question","unsafe_code":false,"validation_split":"train"},"blimp_matrix_question_npi_licensor_present":{"dataset_name":"matrix_question_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_matrix_question_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_1":{"dataset_name":"npi_present_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_1.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_1","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_2":{"dataset_name":"npi_present_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_2.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_2","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_licensor_present":{"dataset_name":"only_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_scope":{"dataset_name":"only_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_passive_1":{"dataset_name":"passive_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_1.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_1","unsafe_code":false,"validation_split":"train"},"blimp_passive_2":{"dataset_name":"passive_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_2.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_c_command":{"dataset_name":"principle_A_c_command","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_c_command.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_c_command","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_1":{"dataset_name":"principle_A_case_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_1.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_2":{"dataset_name":"principle_A_case_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_2.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_1":{"dataset_name":"principle_A_domain_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_1.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_2":{"dataset_name":"principle_A_domain_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_2.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_3":{"dataset_name":"principle_A_domain_3","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_3.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_3","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_reconstruction":{"dataset_name":"principle_A_reconstruction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_reconstruction.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_reconstruction","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_1":{"dataset_name":"regular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_2":{"dataset_name":"regular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_licensor_present":{"dataset_name":"sentential_negation_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_scope":{"dataset_name":"sentential_negation_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_sentential_subject_island":{"dataset_name":"sentential_subject_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_subject_island.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_subject_island","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_1":{"dataset_name":"superlative_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_2":{"dataset_name":"superlative_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_1":{"dataset_name":"tough_vs_raising_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_1.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_1","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_2":{"dataset_name":"tough_vs_raising_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_2.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_2","unsafe_code":false,"validation_split":"train"},"blimp_transitive":{"dataset_name":"transitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/transitive.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_transitive","unsafe_code":false,"validation_split":"train"},"blimp_wh_island":{"dataset_name":"wh_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_island.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_island","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_object_gap":{"dataset_name":"wh_questions_object_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_object_gap.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_object_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap":{"dataset_name":"wh_questions_subject_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap_long_distance":{"dataset_name":"wh_questions_subject_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap":{"dataset_name":"wh_vs_that_no_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap_long_distance":{"dataset_name":"wh_vs_that_no_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap":{"dataset_name":"wh_vs_that_with_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap_long_distance":{"dataset_name":"wh_vs_that_with_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap_long_distance","unsafe_code":false,"validation_split":"train"},"commonsense_qa_text":{"dataset_path":"tau/commonsense_qa","description":"","doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/config/tasks/commonsense_qa_text.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"commonsense_qa_text","training_split":"train","unsafe_code":false,"validation_split":"validation"},"hellaswag":{"dataset_path":"Rowan/hellaswag","description":"","doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_config":{"doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":"<function process_docs at 0x77ee1d9145e0>","sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","process_docs":"def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\\n    def _process_doc(doc):\\n        ctx = doc[\\"ctx_a\\"] + \\" \\" + doc[\\"ctx_b\\"].capitalize()\\n        out_doc = {\\n            \\"query\\": preprocess(doc[\\"activity_label\\"] + \\": \\" + ctx),\\n            \\"choices\\": [preprocess(ending) for ending in doc[\\"endings\\"]],\\n            \\"gold\\": int(doc[\\"label\\"]),\\n        }\\n        return out_doc\\n\\n    return dataset.map(_process_doc)\\n","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"hellaswag","training_split":"train","unsafe_code":false,"validation_split":"validation"},"piqa":{"dataset_path":"baber/piqa","description":"","doc_to_choice":"{{[sol1, sol2]}}","doc_to_decontamination_query":"goal","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{[sol1, sol2]}}","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/piqa/piqa.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"piqa","training_split":"train","unsafe_code":false,"validation_split":"validation"}},"task_versions":{"arc_challenge":1,"arc_easy":1,"blimp":"2.0","blimp_adjunct_island":1,"blimp_anaphor_gender_agreement":1,"blimp_anaphor_number_agreement":1,"blimp_animate_subject_passive":1,"blimp_animate_subject_trans":1,"blimp_causative":1,"blimp_complex_NP_island":1,"blimp_coordinate_structure_constraint_complex_left_branch":1,"blimp_coordinate_structure_constraint_object_extraction":1,"blimp_determiner_noun_agreement_1":1,"blimp_determiner_noun_agreement_2":1,"blimp_determiner_noun_agreement_irregular_1":1,"blimp_determiner_noun_agreement_irregular_2":1,"blimp_determiner_noun_agreement_with_adj_2":1,"blimp_determiner_noun_agreement_with_adj_irregular_1":1,"blimp_determiner_noun_agreement_with_adj_irregular_2":1,"blimp_determiner_noun_agreement_with_adjective_1":1,"blimp_distractor_agreement_relational_noun":1,"blimp_distractor_agreement_relative_clause":1,"blimp_drop_argument":1,"blimp_ellipsis_n_bar_1":1,"blimp_ellipsis_n_bar_2":1,"blimp_existential_there_object_raising":1,"blimp_existential_there_quantifiers_1":1,"blimp_existential_there_quantifiers_2":1,"blimp_existential_there_subject_raising":1,"blimp_expletive_it_object_raising":1,"blimp_inchoative":1,"blimp_intransitive":1,"blimp_irregular_past_participle_adjectives":1,"blimp_irregular_past_participle_verbs":1,"blimp_irregular_plural_subject_verb_agreement_1":1,"blimp_irregular_plural_subject_verb_agreement_2":1,"blimp_left_branch_island_echo_question":1,"blimp_left_branch_island_simple_question":1,"blimp_matrix_question_npi_licensor_present":1,"blimp_npi_present_1":1,"blimp_npi_present_2":1,"blimp_only_npi_licensor_present":1,"blimp_only_npi_scope":1,"blimp_passive_1":1,"blimp_passive_2":1,"blimp_principle_A_c_command":1,"blimp_principle_A_case_1":1,"blimp_principle_A_case_2":1,"blimp_principle_A_domain_1":1,"blimp_principle_A_domain_2":1,"blimp_principle_A_domain_3":1,"blimp_principle_A_reconstruction":1,"blimp_regular_plural_subject_verb_agreement_1":1,"blimp_regular_plural_subject_verb_agreement_2":1,"blimp_sentential_negation_npi_licensor_present":1,"blimp_sentential_negation_npi_scope":1,"blimp_sentential_subject_island":1,"blimp_superlative_quantifiers_1":1,"blimp_superlative_quantifiers_2":1,"blimp_tough_vs_raising_1":1,"blimp_tough_vs_raising_2":1,"blimp_transitive":1,"blimp_wh_island":1,"blimp_wh_questions_object_gap":1,"blimp_wh_questions_subject_gap":1,"blimp_wh_questions_subject_gap_long_distance":1,"blimp_wh_vs_that_no_gap":1,"blimp_wh_vs_that_no_gap_long_distance":1,"blimp_wh_vs_that_with_gap":1,"blimp_wh_vs_that_with_gap_long_distance":1,"commonsense_qa_text":"Yaml","hellaswag":1,"piqa":1}},"rank_range":[19,19],"release_date":"2026-06-19","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/atom-3-4m/20260716T154647Z-v1.1","source":"UniversalComputingResearch/Atom3.4m","track":"base","updated_at":"2026-07-16T15:46:48.573093+00:00","vocab_size":4096},{"artifacts":{"result_file":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt-s-1-4m/20260716T154647Z-v1.1/results_v1.1_merged.json","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt-s-1-4m/20260716T154647Z-v1.1"},"benchmarks":{"arc_challenge":{"accuracy":0.22013651877133106,"chance":0.25,"ci":[-7.053469852104666,-0.6825938566552892],"classification":"below chance","label":"ARC-Challenge","n_samples":1172,"score":-3.9817974971558585},"arc_easy":{"accuracy":0.31776094276094274,"chance":0.25,"ci":[6.509539842873177,11.503928170594838],"classification":"reliably above chance","label":"ARC-Easy","n_samples":2376,"score":9.034792368125698},"blimp":{"accuracy":0.6599056216931216,"categories":{"Anaphor agreement":0.6275,"Argument structure":0.6435555555555555,"Binding":0.7425714285714285,"Control / raising":0.6728000000000001,"Determiner-noun agreement":0.779625,"Ellipsis":0.6525000000000001,"Filler-gap":0.6531428571428571,"Irregular forms":0.8235,"Island effects":0.5603750000000001,"NPI licensing":0.41571428571428576,"Quantifiers":0.6217499999999999,"Subject-verb agreement":0.7258333333333332},"category_count":12,"chance":0.5,"ci":[31.20117526455027,32.64844031084656],"classification":"reliably above chance","label":"BLiMP","n_samples":70000,"score":31.981124338624323},"commonsense_qa":{"accuracy":0.23423423423423423,"chance":0.2,"ci":[1.5125921375921367,7.35309172809171],"classification":"reliably above chance","label":"CommonsenseQA","n_samples":1221,"score":4.279279279279277},"hellaswag":{"accuracy":0.26847241585341564,"chance":0.25,"ci":[1.3075084644493165,3.5919139613622804],"classification":"reliably above chance","label":"HellaSwag","n_samples":10042,"score":2.4629887804554182},"piqa":{"accuracy":0.5516866158868335,"chance":0.5,"ci":[5.658324265505987,14.692600652883549],"classification":"reliably above chance","label":"PIQA","n_samples":1838,"score":10.33732317736671}},"complete":true,"composite":{"ci":[11.210655224075424,13.297313646044007],"classification":"reliably above chance","score":12.287040849624308},"id":"gpt-s-1-4m","name":"GPT-S 1.4M","params_m":1.426,"protocol":{"bootstrap_replicates":2000,"bootstrap_seed":20260716,"harness":"lm-eval==0.4.12","harness_git_hash":null,"inference_config":{"batch_size":"auto","batch_sizes":[64],"bootstrap_iters":100000,"device":"cuda:0","fewshot_seed":1234,"gen_kwargs":{},"limit":null,"model":"hf","model_args":{"dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true},"model_dtype":"torch.bfloat16","model_num_parameters":1426176,"model_revision":"main","model_sha":"","numpy_seed":1234,"random_seed":1234,"tasks":["hellaswag","piqa","arc_easy","arc_challenge","commonsense_qa_text","blimp"],"torch_seed":1234,"use_cache":null,"v1_1_sources":{"rechecked_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt-s-1-4m/20260716T154341Z-v1.1/results_v1.1_merged.json","unchanged_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt-s-1-4m/20260716T154341Z-v1.1/results_v1.1_merged.json"}},"n_shot":{"arc_easy":0,"blimp":0,"blimp_adjunct_island":0,"blimp_anaphor_gender_agreement":0,"blimp_anaphor_number_agreement":0,"blimp_animate_subject_passive":0,"blimp_animate_subject_trans":0,"blimp_causative":0,"blimp_complex_NP_island":0,"blimp_coordinate_structure_constraint_complex_left_branch":0,"blimp_coordinate_structure_constraint_object_extraction":0,"blimp_determiner_noun_agreement_1":0,"blimp_determiner_noun_agreement_2":0,"blimp_determiner_noun_agreement_irregular_1":0,"blimp_determiner_noun_agreement_irregular_2":0,"blimp_determiner_noun_agreement_with_adj_2":0,"blimp_determiner_noun_agreement_with_adj_irregular_1":0,"blimp_determiner_noun_agreement_with_adj_irregular_2":0,"blimp_determiner_noun_agreement_with_adjective_1":0,"blimp_distractor_agreement_relational_noun":0,"blimp_distractor_agreement_relative_clause":0,"blimp_drop_argument":0,"blimp_ellipsis_n_bar_1":0,"blimp_ellipsis_n_bar_2":0,"blimp_existential_there_object_raising":0,"blimp_existential_there_quantifiers_1":0,"blimp_existential_there_quantifiers_2":0,"blimp_existential_there_subject_raising":0,"blimp_expletive_it_object_raising":0,"blimp_inchoative":0,"blimp_intransitive":0,"blimp_irregular_past_participle_adjectives":0,"blimp_irregular_past_participle_verbs":0,"blimp_irregular_plural_subject_verb_agreement_1":0,"blimp_irregular_plural_subject_verb_agreement_2":0,"blimp_left_branch_island_echo_question":0,"blimp_left_branch_island_simple_question":0,"blimp_matrix_question_npi_licensor_present":0,"blimp_npi_present_1":0,"blimp_npi_present_2":0,"blimp_only_npi_licensor_present":0,"blimp_only_npi_scope":0,"blimp_passive_1":0,"blimp_passive_2":0,"blimp_principle_A_c_command":0,"blimp_principle_A_case_1":0,"blimp_principle_A_case_2":0,"blimp_principle_A_domain_1":0,"blimp_principle_A_domain_2":0,"blimp_principle_A_domain_3":0,"blimp_principle_A_reconstruction":0,"blimp_regular_plural_subject_verb_agreement_1":0,"blimp_regular_plural_subject_verb_agreement_2":0,"blimp_sentential_negation_npi_licensor_present":0,"blimp_sentential_negation_npi_scope":0,"blimp_sentential_subject_island":0,"blimp_superlative_quantifiers_1":0,"blimp_superlative_quantifiers_2":0,"blimp_tough_vs_raising_1":0,"blimp_tough_vs_raising_2":0,"blimp_transitive":0,"blimp_wh_island":0,"blimp_wh_questions_object_gap":0,"blimp_wh_questions_subject_gap":0,"blimp_wh_questions_subject_gap_long_distance":0,"blimp_wh_vs_that_no_gap":0,"blimp_wh_vs_that_no_gap_long_distance":0,"blimp_wh_vs_that_with_gap":0,"blimp_wh_vs_that_with_gap_long_distance":0,"commonsense_qa":0,"hellaswag":0,"piqa":0},"suite":"slm-benchmark-v1.1","task_config":{"arc_challenge":{"dataset_name":"ARC-Challenge","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_challenge.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_challenge","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"arc_easy":{"dataset_name":"ARC-Easy","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_easy.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_easy","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"blimp_adjunct_island":{"dataset_name":"adjunct_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/adjunct_island.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_adjunct_island","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_gender_agreement":{"dataset_name":"anaphor_gender_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_gender_agreement","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_number_agreement":{"dataset_name":"anaphor_number_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_number_agreement.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_number_agreement","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_passive":{"dataset_name":"animate_subject_passive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_passive.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_passive","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_trans":{"dataset_name":"animate_subject_trans","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_trans.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_trans","unsafe_code":false,"validation_split":"train"},"blimp_causative":{"dataset_name":"causative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/causative.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_causative","unsafe_code":false,"validation_split":"train"},"blimp_complex_NP_island":{"dataset_name":"complex_NP_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/complex_NP_island.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_complex_NP_island","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_complex_left_branch":{"dataset_name":"coordinate_structure_constraint_complex_left_branch","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_complex_left_branch","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_object_extraction":{"dataset_name":"coordinate_structure_constraint_object_extraction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_object_extraction","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_1":{"dataset_name":"determiner_noun_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_2":{"dataset_name":"determiner_noun_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_1":{"dataset_name":"determiner_noun_agreement_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_2":{"dataset_name":"determiner_noun_agreement_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_2":{"dataset_name":"determiner_noun_agreement_with_adj_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_1":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_2":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adjective_1":{"dataset_name":"determiner_noun_agreement_with_adjective_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adjective_1","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relational_noun":{"dataset_name":"distractor_agreement_relational_noun","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relational_noun","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relative_clause":{"dataset_name":"distractor_agreement_relative_clause","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relative_clause","unsafe_code":false,"validation_split":"train"},"blimp_drop_argument":{"dataset_name":"drop_argument","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/drop_argument.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_drop_argument","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_1":{"dataset_name":"ellipsis_n_bar_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_1","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_2":{"dataset_name":"ellipsis_n_bar_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_object_raising":{"dataset_name":"existential_there_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_object_raising.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_1":{"dataset_name":"existential_there_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_2":{"dataset_name":"existential_there_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_subject_raising":{"dataset_name":"existential_there_subject_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_subject_raising.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_subject_raising","unsafe_code":false,"validation_split":"train"},"blimp_expletive_it_object_raising":{"dataset_name":"expletive_it_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/expletive_it_object_raising.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_expletive_it_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_inchoative":{"dataset_name":"inchoative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/inchoative.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_inchoative","unsafe_code":false,"validation_split":"train"},"blimp_intransitive":{"dataset_name":"intransitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/intransitive.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_intransitive","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_adjectives":{"dataset_name":"irregular_past_participle_adjectives","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_adjectives","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_verbs":{"dataset_name":"irregular_past_participle_verbs","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_verbs","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_1":{"dataset_name":"irregular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_2":{"dataset_name":"irregular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_echo_question":{"dataset_name":"left_branch_island_echo_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_echo_question","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_simple_question":{"dataset_name":"left_branch_island_simple_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_simple_question","unsafe_code":false,"validation_split":"train"},"blimp_matrix_question_npi_licensor_present":{"dataset_name":"matrix_question_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_matrix_question_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_1":{"dataset_name":"npi_present_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_1","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_2":{"dataset_name":"npi_present_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_2","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_licensor_present":{"dataset_name":"only_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_scope":{"dataset_name":"only_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_passive_1":{"dataset_name":"passive_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_1","unsafe_code":false,"validation_split":"train"},"blimp_passive_2":{"dataset_name":"passive_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_c_command":{"dataset_name":"principle_A_c_command","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_c_command.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_c_command","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_1":{"dataset_name":"principle_A_case_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_2":{"dataset_name":"principle_A_case_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_1":{"dataset_name":"principle_A_domain_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_2":{"dataset_name":"principle_A_domain_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_3":{"dataset_name":"principle_A_domain_3","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_3.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_3","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_reconstruction":{"dataset_name":"principle_A_reconstruction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_reconstruction.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_reconstruction","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_1":{"dataset_name":"regular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_2":{"dataset_name":"regular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_licensor_present":{"dataset_name":"sentential_negation_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_scope":{"dataset_name":"sentential_negation_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_sentential_subject_island":{"dataset_name":"sentential_subject_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_subject_island.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_subject_island","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_1":{"dataset_name":"superlative_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_2":{"dataset_name":"superlative_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_1":{"dataset_name":"tough_vs_raising_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_1","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_2":{"dataset_name":"tough_vs_raising_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_2","unsafe_code":false,"validation_split":"train"},"blimp_transitive":{"dataset_name":"transitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/transitive.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_transitive","unsafe_code":false,"validation_split":"train"},"blimp_wh_island":{"dataset_name":"wh_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_island.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_island","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_object_gap":{"dataset_name":"wh_questions_object_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_object_gap.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_object_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap":{"dataset_name":"wh_questions_subject_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap_long_distance":{"dataset_name":"wh_questions_subject_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap":{"dataset_name":"wh_vs_that_no_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap_long_distance":{"dataset_name":"wh_vs_that_no_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap":{"dataset_name":"wh_vs_that_with_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap_long_distance":{"dataset_name":"wh_vs_that_with_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap_long_distance","unsafe_code":false,"validation_split":"train"},"commonsense_qa_text":{"dataset_path":"tau/commonsense_qa","description":"","doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/config/tasks/commonsense_qa_text.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"commonsense_qa_text","training_split":"train","unsafe_code":false,"validation_split":"validation"},"hellaswag":{"dataset_path":"Rowan/hellaswag","description":"","doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_config":{"doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":"<function process_docs at 0x7c83fdb90c20>","sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","process_docs":"def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\\n    def _process_doc(doc):\\n        ctx = doc[\\"ctx_a\\"] + \\" \\" + doc[\\"ctx_b\\"].capitalize()\\n        out_doc = {\\n            \\"query\\": preprocess(doc[\\"activity_label\\"] + \\": \\" + ctx),\\n            \\"choices\\": [preprocess(ending) for ending in doc[\\"endings\\"]],\\n            \\"gold\\": int(doc[\\"label\\"]),\\n        }\\n        return out_doc\\n\\n    return dataset.map(_process_doc)\\n","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"hellaswag","training_split":"train","unsafe_code":false,"validation_split":"validation"},"piqa":{"dataset_path":"baber/piqa","description":"","doc_to_choice":"{{[sol1, sol2]}}","doc_to_decontamination_query":"goal","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{[sol1, sol2]}}","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/piqa/piqa.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"piqa","training_split":"train","unsafe_code":false,"validation_split":"validation"}},"task_versions":{"arc_challenge":1,"arc_easy":1,"blimp":"2.0","blimp_adjunct_island":1,"blimp_anaphor_gender_agreement":1,"blimp_anaphor_number_agreement":1,"blimp_animate_subject_passive":1,"blimp_animate_subject_trans":1,"blimp_causative":1,"blimp_complex_NP_island":1,"blimp_coordinate_structure_constraint_complex_left_branch":1,"blimp_coordinate_structure_constraint_object_extraction":1,"blimp_determiner_noun_agreement_1":1,"blimp_determiner_noun_agreement_2":1,"blimp_determiner_noun_agreement_irregular_1":1,"blimp_determiner_noun_agreement_irregular_2":1,"blimp_determiner_noun_agreement_with_adj_2":1,"blimp_determiner_noun_agreement_with_adj_irregular_1":1,"blimp_determiner_noun_agreement_with_adj_irregular_2":1,"blimp_determiner_noun_agreement_with_adjective_1":1,"blimp_distractor_agreement_relational_noun":1,"blimp_distractor_agreement_relative_clause":1,"blimp_drop_argument":1,"blimp_ellipsis_n_bar_1":1,"blimp_ellipsis_n_bar_2":1,"blimp_existential_there_object_raising":1,"blimp_existential_there_quantifiers_1":1,"blimp_existential_there_quantifiers_2":1,"blimp_existential_there_subject_raising":1,"blimp_expletive_it_object_raising":1,"blimp_inchoative":1,"blimp_intransitive":1,"blimp_irregular_past_participle_adjectives":1,"blimp_irregular_past_participle_verbs":1,"blimp_irregular_plural_subject_verb_agreement_1":1,"blimp_irregular_plural_subject_verb_agreement_2":1,"blimp_left_branch_island_echo_question":1,"blimp_left_branch_island_simple_question":1,"blimp_matrix_question_npi_licensor_present":1,"blimp_npi_present_1":1,"blimp_npi_present_2":1,"blimp_only_npi_licensor_present":1,"blimp_only_npi_scope":1,"blimp_passive_1":1,"blimp_passive_2":1,"blimp_principle_A_c_command":1,"blimp_principle_A_case_1":1,"blimp_principle_A_case_2":1,"blimp_principle_A_domain_1":1,"blimp_principle_A_domain_2":1,"blimp_principle_A_domain_3":1,"blimp_principle_A_reconstruction":1,"blimp_regular_plural_subject_verb_agreement_1":1,"blimp_regular_plural_subject_verb_agreement_2":1,"blimp_sentential_negation_npi_licensor_present":1,"blimp_sentential_negation_npi_scope":1,"blimp_sentential_subject_island":1,"blimp_superlative_quantifiers_1":1,"blimp_superlative_quantifiers_2":1,"blimp_tough_vs_raising_1":1,"blimp_tough_vs_raising_2":1,"blimp_transitive":1,"blimp_wh_island":1,"blimp_wh_questions_object_gap":1,"blimp_wh_questions_subject_gap":1,"blimp_wh_questions_subject_gap_long_distance":1,"blimp_wh_vs_that_no_gap":1,"blimp_wh_vs_that_no_gap_long_distance":1,"blimp_wh_vs_that_with_gap":1,"blimp_wh_vs_that_with_gap_long_distance":1,"commonsense_qa_text":"Yaml","hellaswag":1,"piqa":1}},"rank_range":[20,20],"release_date":"2026-06-01","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt-s-1-4m/20260716T154647Z-v1.1","source":"AxiomicLabs/GPT-S-1.4M","track":"base","updated_at":"2026-07-16T15:46:51.370873+00:00","vocab_size":4096}]'),Yn="slm-benchmark-v1.1",Un={comparisons:Kn,generated_at:$n,models:Hn,suite:Yn},Gn="slm-benchmark-v1.1",Vn="Illustrative data — not an evaluation",Xn=[{id:"demo-110m",name:"Illustrative Base 110M",params_m:110,track:"base",complete:!0,rank_range:[1,2],composite:{score:16.8,ci:[15.1,18.6],classification:"reliably above chance"},benchmarks:{hellaswag:{score:12.4,ci:[10.7,14.2]},piqa:{score:20.6,ci:[17.1,24]},arc_easy:{score:15,ci:[11.5,18.5]},arc_challenge:{score:7.1,ci:[3.4,10.7]},commonsense_qa:{score:8.9,ci:[5.2,12.6]},blimp:{score:27.2,ci:[24.8,29.6]}}},{id:"demo-160m",name:"Illustrative Base 160M",params_m:160,track:"base",complete:!0,rank_range:[1,2],composite:{score:17.7,ci:[16,19.5],classification:"reliably above chance"},benchmarks:{hellaswag:{score:13.9,ci:[12.1,15.6]},piqa:{score:22.1,ci:[18.7,25.4]},arc_easy:{score:16.3,ci:[12.9,19.7]},arc_challenge:{score:9.8,ci:[6.1,13.5]},commonsense_qa:{score:10.6,ci:[7.1,14]},blimp:{score:25.8,ci:[23.5,28.1]}}},{id:"demo-60m",name:"Illustrative Base 60M",params_m:60,track:"base",complete:!0,rank_range:[3,3],composite:{score:5.6,ci:[3.9,7.4],classification:"reliably above chance"},benchmarks:{hellaswag:{score:2.1,ci:[.5,3.8]},piqa:{score:7.3,ci:[4,10.5]},arc_easy:{score:3.8,ci:[.8,6.7]},arc_challenge:{score:1.4,ci:[-2.4,5.2]},commonsense_qa:{score:1.2,ci:[-2.1,4.4]},blimp:{score:13.6,ci:[11.2,16]}}}],Wn={suite:Gn,generated_at:Vn,models:Xn};var eo=X('<meta name="description" content="A carefully scored compact language model benchmark from Universal Computing Research."/>'),bt=X("<button> </button>"),to=X('<colgroup><col class="rank-column"/><col class="model-column"/><col class="release-column"/><col class="parameters-column"/><col class="vocab-column"/><col class="track-column"/><col class="score-column"/><col class="hellaswag-column"/><col class="benchmark-column"/><col class="benchmark-column"/><col class="benchmark-column"/><col class="benchmark-column"/><col class="benchmark-column"/></colgroup>'),_o=X('<th><button class="sort-button" type="button"><!></button></th>'),no=X('<tr class="empty-row"><td>No complete results for this track yet.</td></tr>'),oo=X('<a class="model-link" target="_blank" rel="noreferrer"><span class="model-name"> </span></a>'),io=X('<span class="model-name"> </span>'),ao=X('<span class="model-source"> </span>'),so=X(" <span> </span>",1),lo=X("<td><!></td>"),ro=X('<tr><td class="rank"> </td><td><!> <!></td><td class="metadata"> </td><td> </td><td class="metadata"> </td><td class="track-cell"><span> </span></td><!></tr>'),co=nt('<line class="grid-line"></line><text class="axis-text" text-anchor="end"> </text>',1),mo=nt('<text class="axis-text" text-anchor="middle"> </text>'),po=nt('<line class="frontier-line"></line>'),uo=nt('<circle r="7" tabindex="0" role="button"></circle>'),go=X('<div class="chart-tooltip"><strong> </strong> <span> </span> <b> <em> </em></b></div>'),ho=X('<div class="scatter-area" role="group"><svg preserveAspectRatio="xMidYMid meet" role="img"><text class="axis-title" x="7" text-anchor="middle">Chance-normalized score</text><!><line class="axis-line"></line><!><text class="axis-title" dy="16" text-anchor="middle">Parameters (square-root scale)</text><!><!></svg> <!></div>'),bo=X('<div class="empty-chart">Completed evaluations will appear here.</div>'),fo=X("<i></i>"),yo=X('<div class="bar-row"><span class="bar-rank"> </span> <span class="bar-name"> </span> <div class="bar-track"><i class="zero-line"></i><i></i><!></div> <strong> </strong></div>'),vo=X('<button type="button"><span class="bar-toggle-chevron" aria-hidden="true"></span></button>'),wo=X('<div class="bar-list"></div> <!>',1),xo=X('<div class="empty-chart short">No measured values for this selection.</div>'),ko=X('<main><section class="hero" id="top"><div class="hero-copy"><div class="hero-kicker"><p class="eyebrow">Small language model evaluation</p> <span class="edition">Research index</span></div> <h1>Compact<br/>Language Model<br/><span>Index</span></h1> <p class="hero-description">A fixed evaluation of compact language models across six benchmarks, with chance-normalized scores and 95% intervals</p> <a class="hero-link" href="#leaderboard-title">Explore the leaderboard</a></div></section> <section class="summary-grid" aria-label="Suite summary"><article><span class="summary-index">Evaluation breadth</span><div class="summary-value"><strong>6</strong><span>core benchmarks</span></div></article> <article><span class="summary-index">Linguistic depth</span><div class="summary-value"><strong>12</strong><span>BLiMP categories</span></div></article> <article><span class="summary-index">Current sample</span><div class="summary-value"><strong> </strong><span>models measured</span></div></article></section> <section class="leaderboard-section" aria-labelledby="leaderboard-title"><div class="section-heading table-heading"><div><p class="eyebrow">Ranked results</p> <h2 id="leaderboard-title">Core leaderboard</h2></div> <div class="leaderboard-controls"><div class="filter-stack"><div class="top-filter-row"><div class="filters view-filter"><button type="button">Simplified view</button></div> <div class="filters" aria-label="Model track filter"></div></div> <div class="size-filters" aria-label="Maximum parameter count"></div></div></div></div> <div class="table-wrap"><table><!><thead><tr><th>Rank</th><th><button class="sort-button" type="button">Model</button></th><th class="metadata-head" title="Hugging Face repository creation date">Release date</th><th><button class="sort-button" type="button">Parameters</button></th><th class="metadata-head">Vocab. size</th><th class="track-head">Track</th><th><button class="sort-button" type="button">Score <span>95% CI</span></button></th><!></tr></thead><tbody><!></tbody></table></div> <p class="table-note">ARC-E* = ARC-Easy · ARC-C* = ARC-Challenge · CQA* = CommonsenseQA</p></section> <section class="chart-controls" aria-label="Chart metric selector"><div><p class="eyebrow">Chart metric</p> <p class="section-note">Select a benchmark to update both the scaling chart and score comparison below.</p></div> <div class="task-tabs" aria-label="Benchmark score selector"></div></section> <section class="panel landscape" aria-labelledby="landscape-title"><div class="section-heading"><div><p class="eyebrow">Model scaling</p> <h2 id="landscape-title"> </h2></div> <div class="legend"><span class="legend-dot base"></span>Base <span class="legend-dot instruction"></span>Instruction-tuned <span class="frontier-label"><span class="frontier-key"></span>Pareto trend</span></div></div> <!></section> <section class="panel score-panel" aria-labelledby="scores-title"><div class="section-heading compact-heading"><div><h2 id="scores-title">Model score comparison</h2> <p class="section-note">The top 10 matching models are shown by default. This comparison uses the shared metric selected above.</p></div></div> <!></section> <section class="bottom-grid"><article class="panel method-card"><p class="eyebrow">Scoring method</p> <h3>Chance-normalized composite score</h3> <p class="formula">100 × <span>(accuracy − chance)</span> / <span>(1 − chance)</span></p> <p>For each benchmark, the observed result is normalized against its chance level: chance maps to 0 and a perfect result maps to 100.</p> <p>HellaSwag, PIQA, and CommonsenseQA form a commonsense domain worth 50% of the composite, with each benchmark weighted equally within that domain. Science and world knowledge contribute 25% after combining ARC-Easy (75%) and ARC-Challenge (25%). BLiMP contributes the remaining 25% for grammatical competence and is macro-averaged across its 12 linguistic categories.</p> <p>CommonsenseQA scores the likelihood of each full answer text with length normalization, rather than scoring the A–E answer label. This prevents a model’s letter preference from being mistaken for commonsense capability.</p> <p>Each benchmark interval is a 95% percentile bootstrap. Model comparisons use paired bootstrap resampling on matched evaluation items. Results below chance remain negative, and rank ranges reflect comparisons that do not distinguish neighbouring models.</p></article> <article class="panel protocol-card"><p class="eyebrow">Evaluation protocol</p> <h2>Curated evaluation</h2> <p>CompactLMIndex is a curated benchmark, not an open intake list. We are establishing a Pareto-frontier baseline across model sizes and highlighting training or inference strategies that put a model ahead of its peers at a comparable parameter count.</p> <ul><li>Complete core suite required for a rank</li><li>Zero-shot, non-chat log-likelihood prompts</li><li>Paired bootstrap comparisons on matched questions</li><li>Benchmark-specific training or fine-tuning is discouraged and may exclude a model from comparison</li></ul> <p>Every published result is independently checked through our internal verification process. Final inclusion and presentation decisions remain with Universal Computing Research.</p> <p>If you have interesting results for a model you think is suitable for this benchmark, you can open a discussion on the Space for consideration.</p></article></section> <section class="panel additional-card" aria-labelledby="credits-title"><div><p class="eyebrow">Credits &amp; scope</p> <h2 id="credits-title">Efficiency at smaller scales</h2></div> <div class="credits-content"><p>CompactLMIndex looks at smaller language models. At this scale, architecture, training, and inference choices can make a real difference. Every model follows the same evaluation protocol, so results are compared fairly rather than by size alone.</p> <p class="credits-attribution">Inspired by the <a href="https://huggingface.co/open-llm-leaderboard" target="_blank" rel="noreferrer">Open LLM Leaderboard</a>, <a href="https://huggingface.co/spaces/AxiomicLabs/Open_SLM_Leaderboard" target="_blank" rel="noreferrer">Axiomic Labs Open SLM Leaderboard</a>, and <a href="https://github.com/openai/parameter-golf" target="_blank" rel="noreferrer">OpenAI Parameter Golf</a>.</p></div></section> <footer><p>Compact Language Model Index by <a href="https://huggingface.co/UniversalComputingResearch" target="_blank" rel="noreferrer">Universal Computing Research</a></p></footer></main>');function So(t,n){vn(n,!1);const s=[{id:"average",label:"Average"},{id:"hellaswag",label:"HellaSwag"},{id:"piqa",label:"PIQA"},{id:"arc_easy",label:"ARC-Easy"},{id:"arc_challenge",label:"ARC-Challenge"},{id:"commonsense_qa",label:"CommonsenseQA"},{id:"blimp",label:"BLiMP"}];let c=ee(Un),p=ee("all"),l=ee("all"),a=ee("average"),g=ee(null),y=ee([]),b=ee([]),f=ee(0),C=ee([]),R=ee([]),A=ee(!1),D=ee(!1),J=ee("average"),m=ee("desc"),v=ee([]),U=ee(!1);const M=10,K=o=>Number(o).toFixed(1),ce=o=>Number.isFinite(Number(o))?o>=1e3?`${(o/1e3).toFixed(o>=1e4?1:2)}B`:`${Number(o).toFixed(o<10?2:o<100?1:0)}M`:"—",ie=o=>/^\d{4}-\d{2}-\d{2}$/.test(String(o||""))?o:"—",W=o=>{const _=Number(o);return Number.isFinite(_)&&_>0?_.toLocaleString("en-US"):"—"},ue=o=>{const _=Number(o);return!Number.isFinite(_)||_<10?_:Math.round(_/10)*10},ae=o=>(o==null?void 0:o.length)===2?`${K(o[0])}–${K(o[1])}`:"—",se=o=>{const[_,i]=o.rank_range||[];return _?_===i?`${_}`:`${_}–${i}`:"—"},le=(o,_=e(a))=>{var i;return _==="average"?o.composite:(i=o.benchmarks)==null?void 0:i[_]},Z=(o,_=e(a))=>{var i;return Number((i=le(o,_))==null?void 0:i.score)},fe=o=>{var _;return((_=s.find(i=>i.id===o))==null?void 0:_.label)||"Average"},Re=o=>{const _=String(o.source||"").trim();return _?/^https?:\/\//i.test(_)?_:`https://huggingface.co/${_}`:null},Je=(o,_)=>_==="model"?String(o.name||"").toLocaleLowerCase():_==="parameters"?Number(o.params_m):Z(o,_);function Ee(o){if(e(J)===o){V(m,e(m)==="asc"?"desc":"asc");return}V(J,o),V(m,o==="model"?"asc":"desc")}function Be(o,_,i){const k=i==="asc"?1:-1;return[...o].sort((d,L)=>{const I=Je(d,_),T=Je(L,_),P=I===""||typeof I=="number"&&!Number.isFinite(I),Q=T===""||typeof T=="number"&&!Number.isFinite(T);return P||Q?P===Q?0:P?1:-1:typeof I=="number"&&typeof T=="number"?(I-T)*k:String(I).localeCompare(String(T))*k})}function De(o,_){const i=[...o].sort((d,L)=>Number(d.params_m)-Number(L.params_m));let k=-1/0;return i.filter(d=>{const L=Z(d,_);return!Number.isFinite(L)||L<=k?!1:(k=L,!0)})}function ge(o,_,i){const k=De(o,i);if(k.length<2)return null;const d=k.map(j=>({x:_.parameterCoordinate(j.params_m),y:Z(j,i)})),L=d.reduce((j,z)=>j+z.x,0)/d.length,I=d.reduce((j,z)=>j+z.y,0)/d.length,T=d.reduce((j,z)=>j+(z.x-L)**2,0),P=T?d.reduce((j,z)=>j+(z.x-L)*(z.y-I),0)/T:0,Q=j=>Math.max(_.yMin,Math.min(_.yMax,I+P*(j-L)));return{x1:_.xFromCoordinate(_.xMin),y1:_.y(Q(_.xMin)),x2:_.xFromCoordinate(_.xMax),y2:_.y(Q(_.xMax))}}function Fe(o,_=!1,i="average"){const k=o.map(q=>Z(q,i)).filter(Number.isFinite),d=q=>Math.sqrt(Number(q)),L=o.map(q=>d(q.params_m)),I=Math.max(0,Math.floor(Math.min(...L)*10)/10-.1),T=Math.ceil(Math.max(...L)*10)/10+.1,P=Math.min(-5,Math.floor(Math.min(...k)/5)*5-2),Q=Math.max(10,Math.ceil(Math.max(...k)/5)*5+2),j=_?820:1120,z=390,_e={left:62,right:28,top:28,bottom:52},he=q=>_e.left+(q-I)/(T-I||1)*(j-_e.left-_e.right),ke=q=>he(d(q)),Pe=q=>z-_e.bottom-(q-P)/(Q-P||1)*(z-_e.top-_e.bottom),N=Array.from({length:5},(q,F)=>P+F*(Q-P)/4),w=Array.from({length:4},(q,F)=>I+F*(T-I)/3);return{width:j,height:z,pad:_e,xMin:I,xMax:T,yMin:P,yMax:Q,x:ke,xFromCoordinate:he,y:Pe,yTicks:N,xTicks:w,parameterCoordinate:d}}function xt(o){const _=o.currentTarget,i=_.closest(".scatter-area").getBoundingClientRect(),k=_.getBoundingClientRect(),d=Number.isFinite(o.clientX)?o.clientX:k.left+k.width/2,L=Number.isFinite(o.clientY)?o.clientY:k.top+k.height/2,I=Math.min(112,Math.max(78,(i.width-20)/2));return{x:Math.max(I+10,Math.min(d-i.left,i.width-I-10)),y:Math.max(L-i.top-16,20)}}function kt(o,_){V(g,{model:_,...xt(o)})}function u_(o){e(g)&&V(g,{...e(g),...xt(o)})}L_(()=>{const o=window.matchMedia("(max-width: 760px)"),_=()=>V(D,o.matches);return _(),o.addEventListener("change",_),new URLSearchParams(window.location.search).get("demo")==="1"&&V(c,Wn),()=>o.removeEventListener("change",_)}),Oe(()=>(e(c),e(p),e(l)),()=>{V(y,(e(c).models||[]).filter(o=>(e(p)==="all"||o.track===e(p))&&(e(l)==="all"||ue(o.params_m)<=Number(e(l)))))}),Oe(()=>e(y),()=>{V(b,e(y).filter(o=>o.complete&&Number.isFinite(Number(o.params_m))))}),Oe(()=>(e(b),e(J),e(m)),()=>{V(v,Be(e(b),e(J),e(m)))}),Oe(()=>e(c),()=>{V(f,(e(c).models||[]).filter(o=>o.complete).length)}),Oe(()=>(e(b),e(a)),()=>{V(C,[...e(b)].filter(o=>Number.isFinite(Z(o,e(a)))).sort((o,_)=>Z(_,e(a))-Z(o,e(a))))}),Oe(()=>(e(A),e(C)),()=>{V(R,e(A)?e(C):e(C).slice(0,M))}),wn(),Bn();var ot=ko();Jn("1uha8ag",o=>{var _=eo();m_(()=>{kn.title="Compact Language Model Index"}),O(o,_)});var it=x(h(ot),2),Pt=x(h(it),4),jt=x(h(Pt)),qt=h(jt),g_=h(qt,!0);u(qt),Ge(),u(jt),u(Pt),u(it);var at=x(it,2),st=h(at),Ct=x(h(st),2),Tt=h(Ct),lt=h(Tt),rt=h(lt),ct=h(rt);let Et;u(rt);var St=x(rt,2);we(St,4,()=>[["all","All tracks"],["base","Base"],["instruction","Instruction"]],Se,(o,_)=>{var i=bt();let k;var d=h(i,!0);u(i),Y(()=>{k=te(i,1,"",null,k,{active:e(p)===_[0]}),B(d,r(()=>_[1]))}),ve("click",i,()=>V(p,_[0])),O(o,i)}),u(St),u(lt);var Rt=x(lt,2);we(Rt,4,()=>[["all","All"],["10","<10M"],["50","<50M"],["100","<100M"],["300","<300M"]],Se,(o,_)=>{var i=bt();let k;var d=h(i,!0);u(i),Y(()=>{k=te(i,1,"",null,k,{active:e(l)===_[0]}),B(d,r(()=>_[1]))}),ve("click",i,()=>V(l,_[0])),O(o,i)}),u(Rt),u(Tt),u(Ct),u(st);var zt=x(st,2),mt=h(zt);let Ot;var Jt=h(mt);{var h_=o=>{var _=to();O(o,_)};pe(Jt,o=>{e(U)||o(h_)})}var dt=x(Jt),At=h(dt),Ke=x(h(At));let It;var b_=h(Ke);u(Ke);var $e=x(Ke,2);let Mt;var f_=h($e);u($e);var He=x($e,3);let Lt;var y_=h(He);u(He);var v_=x(He);we(v_,1,()=>r(()=>s.slice(1)),Se,(o,_)=>{var i=_o();let k;var d=h(i),L=h(d);{var I=j=>{var z=Ae("ARC-E*");O(j,z)},T=j=>{var z=Ae("ARC-C*");O(j,z)},P=j=>{var z=Ae("CQA*");O(j,z)},Q=j=>{var z=Ae();Y(()=>B(z,(e(_),r(()=>e(_).label)))),O(j,z)};pe(L,j=>{e(_),r(()=>e(_).id==="arc_easy")?j(I):(e(_),r(()=>e(_).id==="arc_challenge")?j(T,1):(e(_),r(()=>e(_).id==="commonsense_qa")?j(P,2):j(Q,-1)))})}u(d),u(i),Y(()=>{k=te(i,1,"sortable metric-head",null,k,{sorted:e(J)===e(_).id}),S(d,"aria-label",(e(_),r(()=>`Sort by ${e(_).label}`)))}),ve("click",d,()=>Ee(e(_).id)),O(o,i)}),u(At),u(dt);var Qt=x(dt),w_=h(Qt);{var x_=o=>{var _=no(),i=h(_);u(_),Y(()=>S(i,"colspan",e(U)?10:13)),O(o,_)},k_=o=>{var _=M_(),i=Ve(_);we(i,1,()=>e(v),k=>k.id,(k,d)=>{const L=be(()=>(e(d),r(()=>Re(e(d)))));var I=ro(),T=h(I),P=h(T,!0);u(T);var Q=x(T),j=h(Q);{var z=H=>{var $=oo(),de=h($),je=h(de,!0);u(de),u($),Y(()=>{S($,"href",e(L)),S($,"aria-label",(e(d),r(()=>`Open ${e(d).name} on Hugging Face`))),B(je,(e(d),r(()=>e(d).name)))}),O(H,$)},_e=H=>{var $=io(),de=h($,!0);u($),Y(()=>B(de,(e(d),r(()=>e(d).name)))),O(H,$)};pe(j,H=>{e(L)?H(z):H(_e,-1)})}var he=x(j,2);{var ke=H=>{var $=ao(),de=h($,!0);u($),Y(()=>B(de,(e(d),r(()=>e(d).source)))),O(H,$)};pe(he,H=>{e(d),r(()=>e(d).source)&&H(ke)})}u(Q);var Pe=x(Q),N=h(Pe,!0);u(Pe);var w=x(Pe),q=h(w,!0);u(w);var F=x(w),G=h(F,!0);u(F);var re=x(F),me=h(re);let ye;var Ce=h(me,!0);u(me),u(re);var ne=x(re);we(ne,1,()=>s,Se,(H,$,de)=>{const je=be(()=>(e(d),e($),r(()=>le(e(d),e($).id))));var Ye=lo();te(Ye,1,"metric",null,{},{"score-main":de===0});var R_=h(Ye);{var z_=ze=>{var Ue=so(),Ft=Ve(Ue,!0),Kt=x(Ft),J_=h(Kt,!0);u(Kt),Y((A_,I_)=>{B(Ft,A_),B(J_,I_)},[()=>(E(e(je)),r(()=>K(e(je).score))),()=>(E(e(je)),r(()=>ae(e(je).ci)))]),O(ze,Ue)},O_=ze=>{var Ue=Ae("—");O(ze,Ue)};pe(R_,ze=>{e(je)?ze(z_):ze(O_,-1)})}u(Ye),O(H,Ye)}),u(I),Y((H,$,de,je)=>{B(P,H),B(N,$),B(q,de),B(G,je),ye=te(me,1,"track",null,ye,{instruction:e(d).track==="instruction"}),B(Ce,(e(d),r(()=>e(d).track==="instruction"?"instruction":"base")))},[()=>(e(d),r(()=>se(e(d)))),()=>(e(d),r(()=>ie(e(d).release_date))),()=>(e(d),r(()=>ce(e(d).params_m))),()=>(e(d),r(()=>W(e(d).vocab_size)))]),O(k,I)}),O(o,_)};pe(w_,o=>{e(b),r(()=>!e(b).length)?o(x_,1):o(k_,-1)})}u(Qt),u(mt),u(zt),Ge(2),u(at);var pt=x(at,2),Zt=x(h(pt),2);we(Zt,5,()=>s,Se,(o,_)=>{var i=bt();let k;var d=h(i,!0);u(i),Y(()=>{k=te(i,1,"",null,k,{active:e(a)===e(_).id}),B(d,(e(_),r(()=>e(_).label)))}),ve("click",i,()=>V(a,e(_).id)),O(o,i)}),u(Zt),u(pt);var ut=x(pt,2),gt=h(ut),Nt=h(gt),Bt=x(h(Nt),2),P_=h(Bt);u(Bt),u(Nt),Ge(2),u(gt);var j_=x(gt,2);{var q_=o=>{const _=be(()=>e(b)),i=be(()=>(E(e(_)),e(D),e(a),r(()=>Fe(e(_),e(D),e(a))))),k=be(()=>(E(e(_)),E(e(i)),e(a),r(()=>ge(e(_),e(i),e(a)))));var d=ho(),L=h(d),I=h(L),T=x(I);we(T,1,()=>(E(e(i)),r(()=>e(i).yTicks)),Se,(N,w)=>{var q=co(),F=Ve(q),G=x(F),re=h(G,!0);u(G),Y((me,ye,Ce,ne)=>{S(F,"x1",(E(e(i)),r(()=>e(i).pad.left))),S(F,"x2",(E(e(i)),r(()=>e(i).width-e(i).pad.right))),S(F,"y1",me),S(F,"y2",ye),S(G,"x",(E(e(i)),r(()=>e(i).pad.left-14))),S(G,"y",Ce),B(re,ne)},[()=>(E(e(i)),e(w),r(()=>e(i).y(e(w)))),()=>(E(e(i)),e(w),r(()=>e(i).y(e(w)))),()=>(E(e(i)),e(w),r(()=>e(i).y(e(w))+4)),()=>(e(w),r(()=>K(e(w))))]),O(N,q)});var P=x(T),Q=x(P);we(Q,1,()=>(E(e(i)),r(()=>e(i).xTicks)),Se,(N,w)=>{var q=mo(),F=h(q,!0);u(q),Y((G,re)=>{S(q,"x",G),S(q,"y",(E(e(i)),r(()=>e(i).height-17))),B(F,re)},[()=>(E(e(i)),e(w),r(()=>e(i).xFromCoordinate(e(w)))),()=>(e(w),r(()=>ce(e(w)**2)))]),O(N,q)});var j=x(Q),z=x(j);{var _e=N=>{var w=po();Y(()=>{S(w,"x1",(E(e(k)),r(()=>e(k).x1))),S(w,"y1",(E(e(k)),r(()=>e(k).y1))),S(w,"x2",(E(e(k)),r(()=>e(k).x2))),S(w,"y2",(E(e(k)),r(()=>e(k).y2)))}),O(N,w)};pe(z,N=>{e(k)&&N(_e)})}var he=x(z);we(he,1,()=>e(_),N=>N.id,(N,w)=>{var q=uo();let F;Y((G,re,me)=>{F=te(q,0,"point",null,F,{"instruction-point":e(w).track==="instruction"}),S(q,"cx",G),S(q,"cy",re),S(q,"aria-label",me)},[()=>(E(e(i)),e(w),r(()=>e(i).x(e(w).params_m))),()=>(E(e(i)),e(w),e(a),r(()=>e(i).y(Z(e(w),e(a))))),()=>(e(w),e(a),r(()=>`${e(w).name}: ${ce(e(w).params_m)}, ${fe(e(a))} score ${K(Z(e(w),e(a)))}`))]),Xe("mouseenter",q,G=>kt(G,e(w))),ve("mousemove",q,u_),Xe("mouseleave",q,()=>V(g,null)),Xe("focus",q,G=>kt(G,e(w))),Xe("blur",q,()=>V(g,null)),O(N,q)}),u(L);var ke=x(L,2);{var Pe=N=>{var w=go(),q=h(w),F=h(q,!0);u(q);var G=x(q,2),re=h(G);u(G);var me=x(G,2),ye=h(me),Ce=x(ye),ne=h(Ce);u(Ce),u(me),u(w),Y((H,$,de)=>{et(w,(e(g),r(()=>`left:${e(g).x}px; top:${e(g).y}px`))),B(F,(e(g),r(()=>e(g).model.name))),B(re,`${H??""} parameters`),B(ye,`${$??""} `),B(ne,`${de??""} score`)},[()=>(e(g),r(()=>ce(e(g).model.params_m))),()=>(e(g),e(a),r(()=>K(Z(e(g).model,e(a))))),()=>(e(a),r(()=>fe(e(a))))]),O(N,w)};pe(ke,N=>{e(g)&&N(Pe)})}u(d),Y((N,w)=>{S(d,"aria-label",N),S(L,"viewBox",(E(e(i)),r(()=>`0 0 ${e(i).width} ${e(i).height}`))),S(L,"aria-label",w),S(I,"y",(E(e(i)),r(()=>e(i).height/2))),S(I,"transform",(E(e(i)),r(()=>`rotate(-90 7 ${e(i).height/2})`))),S(P,"x1",(E(e(i)),r(()=>e(i).pad.left))),S(P,"x2",(E(e(i)),r(()=>e(i).width-e(i).pad.right))),S(P,"y1",(E(e(i)),r(()=>e(i).height-e(i).pad.bottom))),S(P,"y2",(E(e(i)),r(()=>e(i).height-e(i).pad.bottom))),S(j,"x",(E(e(i)),r(()=>e(i).width/2))),S(j,"y",(E(e(i)),r(()=>e(i).height-1)))},[()=>(e(a),r(()=>`Interactive ${fe(e(a))} score scatter chart`)),()=>(e(a),r(()=>`Scatter chart of model size and chance-normalized ${fe(e(a))} score`))]),O(o,d)},C_=o=>{var _=bo();O(o,_)};pe(j_,o=>{e(b),r(()=>e(b).length)?o(q_):o(C_,-1)})}u(ut);var Dt=x(ut,2),T_=x(h(Dt),2);{var E_=o=>{const _=be(()=>(e(C),e(a),r(()=>Math.min(0,...e(C).flatMap(T=>{var P,Q;return[Z(T),((Q=(P=le(T,e(a)))==null?void 0:P.ci)==null?void 0:Q[0])??Z(T)]}))))),i=be(()=>(e(C),e(a),r(()=>Math.max(10,...e(C).flatMap(T=>{var P,Q;return[Z(T),((Q=(P=le(T,e(a)))==null?void 0:P.ci)==null?void 0:Q[1])??Z(T)]})))));var k=wo(),d=Ve(k);we(d,7,()=>e(R),T=>T.id,(T,P,Q)=>{const j=be(()=>(e(P),r(()=>Z(e(P))))),z=be(()=>(e(P),e(a),r(()=>{var ne;return(ne=le(e(P),e(a)))==null?void 0:ne.ci}))),_e=be(()=>(e(j)-e(_))/(e(i)-e(_)||1)*100);var he=yo(),ke=h(he),Pe=h(ke,!0);u(ke);var N=x(ke,2),w=h(N,!0);u(N);var q=x(N,2),F=x(h(q));let G;var re=x(F);{var me=ne=>{var H=fo();let $;Y(()=>{$=te(H,1,"ci-band",null,$,{"instruction-band":e(P).track==="instruction","negative-band":e(j)<0}),et(H,(E(e(z)),E(e(_)),E(e(i)),r(()=>`--start:${(e(z)[0]-e(_))/(e(i)-e(_)||1)*100}%; --width:${(e(z)[1]-e(z)[0])/(e(i)-e(_)||1)*100}%`)))}),O(ne,H)};pe(re,ne=>{e(z)&&ne(me)})}u(q);var ye=x(q,2),Ce=h(ye,!0);u(ye),u(he),Y((ne,H)=>{B(Pe,e(Q)+1),S(N,"title",(e(P),r(()=>e(P).name))),B(w,(e(P),r(()=>e(P).name))),G=te(F,1,"bar-fill",null,G,{negative:e(j)<0,"instruction-fill":e(P).track==="instruction"}),et(F,ne),B(Ce,H)},[()=>(E(e(j)),E(e(_e)),E(e(_)),E(e(i)),r(()=>`--start:${e(j)<0?e(_e):(0-e(_))/(e(i)-e(_)||1)*100}%; --width:${Math.abs(e(j))/(e(i)-e(_)||1)*100}%`)),()=>(E(e(j)),r(()=>K(e(j))))]),On(3,he,()=>Fn,()=>({duration:180})),O(T,he)}),u(d);var L=x(d,2);{var I=T=>{var P=vo();let Q;Y(()=>{Q=te(P,1,"bar-toggle",null,Q,{expanded:e(A)}),S(P,"aria-expanded",e(A)),S(P,"aria-label",(e(A),e(C),r(()=>e(A)?`Show top ${M} models`:`Show all ${e(C).length} models`))),S(P,"title",(e(A),e(C),r(()=>e(A)?`Show top ${M} models`:`Show all ${e(C).length} models`)))}),ve("click",P,()=>V(A,!e(A))),O(T,P)};pe(L,T=>{e(C),r(()=>e(C).length>M)&&T(I)})}Y(()=>et(d,`--zero:${(0-e(_))/(e(i)-e(_)||1)*100}%`)),O(o,k)},S_=o=>{var _=xo();O(o,_)};pe(T_,o=>{e(R),r(()=>e(R).length)?o(E_):o(S_,-1)})}u(Dt),Ge(6),u(ot),Y(o=>{B(g_,e(f)),S(ct,"aria-pressed",e(U)),Et=te(ct,1,"",null,Et,{active:e(U)}),Ot=te(mt,1,"",null,Ot,{"full-table":!e(U),simplified:e(U)}),It=te(Ke,1,"sortable",null,It,{sorted:e(J)==="model"}),Mt=te($e,1,"sortable",null,Mt,{sorted:e(J)==="parameters"}),Lt=te(He,1,"sortable score-head",null,Lt,{sorted:e(J)==="average"}),B(P_,`Parameters and ${o??""} score`)},[()=>(e(a),r(()=>fe(e(a))))]),ve("click",ct,()=>V(U,!e(U))),ve("click",b_,()=>Ee("model")),ve("click",f_,()=>Ee("parameters")),ve("click",y_,()=>Ee("average")),O(t,ot),xn()}jn(["click","mousemove"]);export{So as component};