{
  "rows": {
    "bert-base": {
      "STS-B": {
        "spearman": 0.4729110370627617,
        "self_cos": 0.6071038246154785,
        "erank": 32.78397076041497
      },
      "SICK-R": {
        "spearman": 0.5864510093336309,
        "self_cos": 0.7215006351470947,
        "erank": 37.94385125683128
      },
      "STS12": {
        "spearman": 0.3086988833433406,
        "self_cos": 0.5711537599563599,
        "erank": 24.885488457022806
      },
      "STS13": {
        "spearman": 0.598789698358598,
        "self_cos": 0.543938159942627,
        "erank": 30.86986022276995
      },
      "STS14": {
        "spearman": 0.4772790322339173,
        "self_cos": 0.5500277280807495,
        "erank": 43.47942126287742
      },
      "STS15": {
        "spearman": 0.6028567377934425,
        "self_cos": 0.5368884801864624,
        "erank": 31.841449950380582
      },
      "STS16": {
        "spearman": 0.6373258795282091,
        "self_cos": 0.5254936218261719,
        "erank": 36.41999517850558
      },
      "BIOSSES": {
        "spearman": 0.5467413735357435,
        "self_cos": 0.8057912588119507,
        "erank": 22.247537230877278
      }
    },
    "ModernBERT-base": {
      "STS-B": {
        "spearman": 0.4214659933378143,
        "self_cos": 0.9001362919807434,
        "erank": 26.060701940886688
      },
      "SICK-R": {
        "spearman": 0.5479313539236145,
        "self_cos": 0.9264936447143555,
        "erank": 33.772658699650634
      },
      "STS12": {
        "spearman": 0.35267588032012254,
        "self_cos": 0.899753749370575,
        "erank": 23.830631594447453
      },
      "STS13": {
        "spearman": 0.42466045764898774,
        "self_cos": 0.8737303018569946,
        "erank": 27.05837292046762
      },
      "STS14": {
        "spearman": 0.37950256732888316,
        "self_cos": 0.886877179145813,
        "erank": 35.62235111202197
      },
      "STS15": {
        "spearman": 0.5348972735706118,
        "self_cos": 0.8894891738891602,
        "erank": 27.04410063335044
      },
      "STS16": {
        "spearman": 0.41738325520770825,
        "self_cos": 0.8923095464706421,
        "erank": 30.73751078648108
      },
      "BIOSSES": {
        "spearman": 0.56299475592828,
        "self_cos": 0.9297112226486206,
        "erank": 25.182114423236325
      }
    },
    "roberta-base": {
      "STS-B": {
        "spearman": 0.543552857620476,
        "self_cos": 0.9594460129737854,
        "erank": 19.810673654998062
      },
      "SICK-R": {
        "spearman": 0.6296135848604789,
        "self_cos": 0.9708269238471985,
        "erank": 28.687218599255942
      },
      "STS12": {
        "spearman": 0.32107583300958337,
        "self_cos": 0.96092689037323,
        "erank": 16.64961051635458
      },
      "STS13": {
        "spearman": 0.5630694228768335,
        "self_cos": 0.9497929215431213,
        "erank": 17.717675551658246
      },
      "STS14": {
        "spearman": 0.4521989441837469,
        "self_cos": 0.9581155776977539,
        "erank": 31.261791973792803
      },
      "STS15": {
        "spearman": 0.6134270059514074,
        "self_cos": 0.9566895961761475,
        "erank": 23.156222890974327
      },
      "STS16": {
        "spearman": 0.6197650044950626,
        "self_cos": 0.9553296566009521,
        "erank": 21.02615146228128
      },
      "BIOSSES": {
        "spearman": 0.5776697891040469,
        "self_cos": 0.9792889952659607,
        "erank": 22.523762624240604
      }
    },
    "albert-base-v2": {
      "STS-B": {
        "spearman": 0.47838740356988224,
        "self_cos": 0.7473497986793518,
        "erank": 20.900041297930812
      },
      "SICK-R": {
        "spearman": 0.5364143671913744,
        "self_cos": 0.8406326770782471,
        "erank": 29.938170627653616
      },
      "STS12": {
        "spearman": 0.31013914440381385,
        "self_cos": 0.7627694010734558,
        "erank": 16.00922378532872
      },
      "STS13": {
        "spearman": 0.483146267057056,
        "self_cos": 0.7211617827415466,
        "erank": 17.203320478364503
      },
      "STS14": {
        "spearman": 0.38088774762981314,
        "self_cos": 0.7328482866287231,
        "erank": 20.45641300099336
      },
      "STS15": {
        "spearman": 0.5542365388896877,
        "self_cos": 0.7444789409637451,
        "erank": 17.547509929381988
      },
      "STS16": {
        "spearman": 0.5490783560304818,
        "self_cos": 0.7460991740226746,
        "erank": 18.88065266457433
      },
      "BIOSSES": {
        "spearman": 0.48625806767174556,
        "self_cos": 0.8894421458244324,
        "erank": 21.42771786935271
      }
    },
    "distilbert": {
      "STS-B": {
        "spearman": 0.5716824380655484,
        "self_cos": 0.6919586658477783,
        "erank": 31.08277638663469
      },
      "SICK-R": {
        "spearman": 0.6423564551561864,
        "self_cos": 0.7864170670509338,
        "erank": 35.077927161990885
      },
      "STS12": {
        "spearman": 0.4343633145261148,
        "self_cos": 0.6602542400360107,
        "erank": 22.28248223724889
      },
      "STS13": {
        "spearman": 0.6490434554954403,
        "self_cos": 0.6435154676437378,
        "erank": 25.445791829998033
      },
      "STS14": {
        "spearman": 0.5409727505617253,
        "self_cos": 0.6495474576950073,
        "erank": 35.54453592949777
      },
      "STS15": {
        "spearman": 0.6662531121156234,
        "self_cos": 0.6480947136878967,
        "erank": 27.60964656786538
      },
      "STS16": {
        "spearman": 0.6853934174177524,
        "self_cos": 0.6376146078109741,
        "erank": 26.913566855735947
      },
      "BIOSSES": {
        "spearman": 0.5162226058557621,
        "self_cos": 0.8552691340446472,
        "erank": 19.53449548926752
      }
    },
    "all-MiniLM-L6-v2": {
      "STS-B": {
        "spearman": 0.8203246731235654,
        "self_cos": 0.025109656155109406,
        "erank": 86.74223196729439
      },
      "SICK-R": {
        "spearman": 0.7758245724449628,
        "self_cos": 0.06721934676170349,
        "erank": 53.173608410222116
      },
      "STS12": {
        "spearman": 0.7236776011707667,
        "self_cos": 0.03342053294181824,
        "erank": 73.90645784660268
      },
      "STS13": {
        "spearman": 0.8058488275444974,
        "self_cos": 0.057570312172174454,
        "erank": 78.58656500093494
      },
      "STS14": {
        "spearman": 0.7558896510296685,
        "self_cos": 0.04318859428167343,
        "erank": 100.1291172830392
      },
      "STS15": {
        "spearman": 0.8538974664370131,
        "self_cos": 0.07552520185709,
        "erank": 51.13212785198057
      },
      "STS16": {
        "spearman": 0.7898903752156878,
        "self_cos": 0.046465590596199036,
        "erank": 88.79120481420858
      },
      "BIOSSES": {
        "spearman": 0.8143800019839345,
        "self_cos": 0.26674872636795044,
        "erank": 38.14934389954246
      }
    },
    "captionbert-v2": {
      "STS-B": {
        "spearman": 0.574721280584061,
        "self_cos": 0.13962045311927795,
        "erank": 36.611595724368264
      },
      "SICK-R": {
        "spearman": 0.6526021262773363,
        "self_cos": 0.3233732581138611,
        "erank": 39.12520142715301
      },
      "STS12": {
        "spearman": 0.505100243433651,
        "self_cos": 0.15351814031600952,
        "erank": 25.25298589643048
      },
      "STS13": {
        "spearman": 0.5994828042362372,
        "self_cos": 0.15675021708011627,
        "erank": 21.999296364515125
      },
      "STS14": {
        "spearman": 0.5451919695494798,
        "self_cos": 0.118232861161232,
        "erank": 36.6908687709357
      },
      "STS15": {
        "spearman": 0.7136498101249104,
        "self_cos": 0.15519322454929352,
        "erank": 28.501096213319794
      },
      "STS16": {
        "spearman": 0.6776026013744647,
        "self_cos": 0.15450555086135864,
        "erank": 31.8939612240238
      },
      "BIOSSES": {
        "spearman": 0.5932785784958415,
        "self_cos": 0.5799180269241333,
        "erank": 24.427896085902894
      }
    },
    "captionbert-v2 + arms": {
      "STS-B": {
        "spearman": 0.7683668046957611,
        "self_cos": 0.09640415757894516,
        "erank": 57.575776091792314
      },
      "SICK-R": {
        "spearman": 0.739122288023732,
        "self_cos": 0.15953481197357178,
        "erank": 35.991980573254025
      },
      "STS12": {
        "spearman": 0.668246368890754,
        "self_cos": 0.1056038960814476,
        "erank": 53.02433223890012
      },
      "STS13": {
        "spearman": 0.7557315383894732,
        "self_cos": 0.1316332221031189,
        "erank": 36.461354336652235
      },
      "STS14": {
        "spearman": 0.6920855847193758,
        "self_cos": 0.09459435939788818,
        "erank": 58.710654139131236
      },
      "STS15": {
        "spearman": 0.8055463362271912,
        "self_cos": 0.13524731993675232,
        "erank": 34.37401499240861
      },
      "STS16": {
        "spearman": 0.7626482895758635,
        "self_cos": 0.12244462221860886,
        "erank": 53.004054443029155
      },
      "BIOSSES": {
        "spearman": 0.6382494827064735,
        "self_cos": 0.4860020875930786,
        "erank": 30.829771534790567
      }
    },
    "captionbert-b": {
      "STS-B": {
        "spearman": 0.5752034051771185,
        "self_cos": 0.141144260764122,
        "erank": 36.09847902658138
      },
      "SICK-R": {
        "spearman": 0.6548215896039966,
        "self_cos": 0.32319971919059753,
        "erank": 39.71071351231075
      },
      "STS12": {
        "spearman": 0.5011826577733569,
        "self_cos": 0.15315845608711243,
        "erank": 24.67258341649139
      },
      "STS13": {
        "spearman": 0.6037065379973308,
        "self_cos": 0.15690317749977112,
        "erank": 21.59487360473189
      },
      "STS14": {
        "spearman": 0.5470297609965277,
        "self_cos": 0.11881668865680695,
        "erank": 36.458411704485556
      },
      "STS15": {
        "spearman": 0.7145781908913957,
        "self_cos": 0.15444867312908173,
        "erank": 28.13897733732091
      },
      "STS16": {
        "spearman": 0.6782449845451882,
        "self_cos": 0.15363498032093048,
        "erank": 31.736178287294557
      },
      "BIOSSES": {
        "spearman": 0.5499763870067562,
        "self_cos": 0.5835995674133301,
        "erank": 25.359985480191284
      }
    },
    "captionbert-b + arms": {
      "STS-B": {
        "spearman": 0.7675048474087047,
        "self_cos": 0.09836453199386597,
        "erank": 55.48125996351648
      },
      "SICK-R": {
        "spearman": 0.7373729049554012,
        "self_cos": 0.155704066157341,
        "erank": 35.30222741160654
      },
      "STS12": {
        "spearman": 0.6705491654312042,
        "self_cos": 0.10657862573862076,
        "erank": 51.06537851244605
      },
      "STS13": {
        "spearman": 0.7381151114408091,
        "self_cos": 0.14639371633529663,
        "erank": 33.99815967138951
      },
      "STS14": {
        "spearman": 0.6944540661227268,
        "self_cos": 0.09880728274583817,
        "erank": 57.450756193664006
      },
      "STS15": {
        "spearman": 0.8108616295716305,
        "self_cos": 0.15598554909229279,
        "erank": 35.80340205852051
      },
      "STS16": {
        "spearman": 0.7695018825173098,
        "self_cos": 0.13625913858413696,
        "erank": 52.54922090549884
      },
      "BIOSSES": {
        "spearman": 0.6471894454457336,
        "self_cos": 0.46235573291778564,
        "erank": 32.9501082390472
      }
    },
    "alephlm0-a1_anchored-s0 [cls]": {
      "STS-B": {
        "spearman": 0.573071088105736,
        "self_cos": 0.14350926876068115,
        "erank": 35.50144406128079
      },
      "SICK-R": {
        "spearman": 0.6528405854409415,
        "self_cos": 0.3282352685928345,
        "erank": 38.614619825229354
      },
      "STS12": {
        "spearman": 0.4995673104846518,
        "self_cos": 0.15833249688148499,
        "erank": 24.129452377848093
      },
      "STS13": {
        "spearman": 0.6013820958824723,
        "self_cos": 0.15985272824764252,
        "erank": 21.625107292530462
      },
      "STS14": {
        "spearman": 0.54573202195453,
        "self_cos": 0.12105981260538101,
        "erank": 35.773891034021375
      },
      "STS15": {
        "spearman": 0.712098362102062,
        "self_cos": 0.15739022195339203,
        "erank": 27.954601760968337
      },
      "STS16": {
        "spearman": 0.6760697167506455,
        "self_cos": 0.15865075588226318,
        "erank": 31.238976948830043
      },
      "BIOSSES": {
        "spearman": 0.5639164636770043,
        "self_cos": 0.5731398463249207,
        "erank": 25.086680621241996
      }
    },
    "alephlm0-a1_anchored-s0 OFF [cls]": {
      "STS-B": {
        "spearman": 0.5809001831745922,
        "self_cos": 0.23671430349349976,
        "erank": 43.590664823105044
      },
      "SICK-R": {
        "spearman": 0.6225928028341274,
        "self_cos": 0.3692883253097534,
        "erank": 42.88240456121129
      },
      "STS12": {
        "spearman": 0.5199742217473295,
        "self_cos": 0.30195584893226624,
        "erank": 26.346809425547928
      },
      "STS13": {
        "spearman": 0.5719708537654687,
        "self_cos": 0.2987832725048065,
        "erank": 31.16828849074626
      },
      "STS14": {
        "spearman": 0.5553630244689209,
        "self_cos": 0.2424396574497223,
        "erank": 48.42436115403721
      },
      "STS15": {
        "spearman": 0.6772577837474945,
        "self_cos": 0.32620739936828613,
        "erank": 36.22237103029055
      },
      "STS16": {
        "spearman": 0.6231575569132095,
        "self_cos": 0.30862316489219666,
        "erank": 36.286747418282225
      },
      "BIOSSES": {
        "spearman": 0.4433896210055891,
        "self_cos": 0.6539771556854248,
        "erank": 28.786422346917345
      }
    },
    "alephlm0-a2_dense-s0 [cls]": {
      "STS-B": {
        "spearman": 0.573072366324965,
        "self_cos": 0.14419333636760712,
        "erank": 35.98422762570628
      },
      "SICK-R": {
        "spearman": 0.6506668073203276,
        "self_cos": 0.32836371660232544,
        "erank": 38.4526017457717
      },
      "STS12": {
        "spearman": 0.49650066767230716,
        "self_cos": 0.15847142040729523,
        "erank": 24.78090493488994
      },
      "STS13": {
        "spearman": 0.5961926058769006,
        "self_cos": 0.1592738926410675,
        "erank": 21.620262873749315
      },
      "STS14": {
        "spearman": 0.5429655185076202,
        "self_cos": 0.12212906032800674,
        "erank": 36.02135618765891
      },
      "STS15": {
        "spearman": 0.7129909416046672,
        "self_cos": 0.15673181414604187,
        "erank": 28.21602101245322
      },
      "STS16": {
        "spearman": 0.6797836555419772,
        "self_cos": 0.15749043226242065,
        "erank": 31.21942284635101
      },
      "BIOSSES": {
        "spearman": 0.5683322269437682,
        "self_cos": 0.5842017531394958,
        "erank": 25.217301887739335
      }
    },
    "alephlm0-a3_random-s0 [cls]": {
      "STS-B": {
        "spearman": 0.5706771552933212,
        "self_cos": 0.14352479577064514,
        "erank": 35.55978405703753
      },
      "SICK-R": {
        "spearman": 0.6538160109623418,
        "self_cos": 0.32862553000450134,
        "erank": 38.30270369633366
      },
      "STS12": {
        "spearman": 0.49656081962376936,
        "self_cos": 0.1591401994228363,
        "erank": 24.59224995124529
      },
      "STS13": {
        "spearman": 0.5998418714690633,
        "self_cos": 0.1619904488325119,
        "erank": 21.432336605232127
      },
      "STS14": {
        "spearman": 0.5447575915629577,
        "self_cos": 0.121549591422081,
        "erank": 35.33897438482058
      },
      "STS15": {
        "spearman": 0.7102527764861621,
        "self_cos": 0.15813423693180084,
        "erank": 28.61807704972682
      },
      "STS16": {
        "spearman": 0.6805651615188786,
        "self_cos": 0.15982232987880707,
        "erank": 30.570304373019894
      },
      "BIOSSES": {
        "spearman": 0.5702539574786901,
        "self_cos": 0.5777398347854614,
        "erank": 24.753200853233796
      }
    },
    "alephlm0-a3_random-s0 OFF [cls]": {
      "STS-B": {
        "spearman": 0.5767135181140858,
        "self_cos": 0.14212168753147125,
        "erank": 44.25351962093043
      },
      "SICK-R": {
        "spearman": 0.6308633779110203,
        "self_cos": 0.2985028624534607,
        "erank": 41.52728649663491
      },
      "STS12": {
        "spearman": 0.49480358844045863,
        "self_cos": 0.18840818107128143,
        "erank": 28.696343444955414
      },
      "STS13": {
        "spearman": 0.573497954089293,
        "self_cos": 0.22091510891914368,
        "erank": 27.41034699256048
      },
      "STS14": {
        "spearman": 0.5485855356144953,
        "self_cos": 0.15782061219215393,
        "erank": 45.81790091859102
      },
      "STS15": {
        "spearman": 0.6850947914443487,
        "self_cos": 0.2140824794769287,
        "erank": 36.87897890029204
      },
      "STS16": {
        "spearman": 0.6306894346157562,
        "self_cos": 0.20071080327033997,
        "erank": 37.9768859805789
      },
      "BIOSSES": {
        "spearman": 0.4776132923814231,
        "self_cos": 0.5308424234390259,
        "erank": 31.442570804915718
      }
    }
  },
  "params": {
    "bert-base": 109482240,
    "ModernBERT-base": 149014272,
    "roberta-base": 124645632,
    "albert-base-v2": 11683584,
    "distilbert": 66362880,
    "all-MiniLM-L6-v2": 22713216,
    "captionbert-v2": 58308864,
    "captionbert-v2 + arms": 63226428,
    "captionbert-b": 58308864,
    "captionbert-b + arms": 63226428,
    "alephlm0-a1_anchored-s0 [cls]": 58345764,
    "alephlm0-a1_anchored-s0 OFF [cls]": 58345764,
    "alephlm0-a2_dense-s0 [cls]": 58308864,
    "alephlm0-a3_random-s0 [cls]": 58345764,
    "alephlm0-a3_random-s0 OFF [cls]": 58345764
  },
  "geometry": {
    "alephlm0-a1_anchored-s0 [cls]": {
      "self_cos": 0.14350926876068115,
      "erank": 35.50144406128079
    },
    "alephlm0-a1_anchored-s0 OFF [cls]": {
      "self_cos": 0.23671430349349976,
      "erank": 43.590664823105044
    },
    "alephlm0-a2_dense-s0 [cls]": {
      "self_cos": 0.14419333636760712,
      "erank": 35.98422762570628
    },
    "alephlm0-a3_random-s0 [cls]": {
      "self_cos": 0.14352479577064514,
      "erank": 35.55978405703753
    },
    "alephlm0-a3_random-s0 OFF [cls]": {
      "self_cos": 0.14212168753147125,
      "erank": 44.25351962093043
    }
  },
  "config": {
    "teachers": [
      [
        "bert-base",
        "google-bert/bert-base-uncased"
      ],
      [
        "ModernBERT-base",
        "answerdotai/ModernBERT-base"
      ],
      [
        "roberta-base",
        "FacebookAI/roberta-base"
      ],
      [
        "albert-base-v2",
        "albert/albert-base-v2"
      ],
      [
        "distilbert",
        "distilbert/distilbert-base-uncased"
      ]
    ],
    "references": [
      [
        "all-MiniLM-L6-v2",
        "sentence-transformers/all-MiniLM-L6-v2"
      ]
    ],
    "trunks": [
      [
        "captionbert-v2",
        "AbstractPhil/captionbert-8192-v2",
        "checkpoints/best_model.pt",
        "amoe/collective",
        "amoe/collective/captionbert-v2-collective.dispatch.pt"
      ],
      [
        "captionbert-b",
        "AbstractPhil/captionbert-8192-v2-B",
        "checkpoints/final_model.pt",
        "amoe/b-collective",
        "amoe/b-collective/captionbert-b-arms-native.dispatch.pt"
      ]
    ],
    "vocab_size": 30522,
    "d_model": 512,
    "n_heads": 8,
    "n_layers": 12,
    "d_ff": 2048,
    "output_dim": 768,
    "max_len": 8192,
    "pooling": "mean",
    "n_slots": 16,
    "K": 64,
    "D": 4,
    "tau": 0.1,
    "hidden": 178,
    "gate_init": -3.0,
    "align_emb": 64,
    "tasks": [
      [
        "STS-B",
        "mteb/stsbenchmark-sts"
      ],
      [
        "SICK-R",
        "mteb/sickr-sts"
      ],
      [
        "STS12",
        "mteb/sts12-sts"
      ],
      [
        "STS13",
        "mteb/sts13-sts"
      ],
      [
        "STS14",
        "mteb/sts14-sts"
      ],
      [
        "STS15",
        "mteb/sts15-sts"
      ],
      [
        "STS16",
        "mteb/sts16-sts"
      ],
      [
        "BIOSSES",
        "mteb/biosses-sts"
      ]
    ],
    "batch_size": 256,
    "max_tokens": 64,
    "geom_n": 2000,
    "seed": 0,
    "out_json": "full_benchmark.json",
    "out_md": "benchmark_tables.md",
    "hf_push": false,
    "hf_repos": [
      "AbstractPhil/captionbert-8192-v2",
      "AbstractPhil/captionbert-8192-v2-B"
    ],
    "hf_path": "eval"
  }
}