File size: 10,395 Bytes
9000bbd | 1 | {"domains":[{"key":"imdb","label":"IMDB","topic":"movies","length":"long","blurb":"Full-length movie reviews from IMDB (Maas et al., 2011)."},{"key":"sst2","label":"SST-2","topic":"movies","length":"short","blurb":"Single-sentence movie-review snippets from the Stanford Sentiment Treebank."},{"key":"yelp","label":"Yelp","topic":"commerce","length":"long","blurb":"Business and restaurant reviews from the Yelp polarity set (Zhang et al., 2015)."},{"key":"amazon","label":"Amazon","topic":"commerce","length":"long","blurb":"Product reviews from the Amazon polarity set (Zhang et al., 2015)."}],"methods":[{"key":"tfidf","label":"TF-IDF + LR","blurb":"No pretraining at all: word/char n-grams into logistic regression. The control."},{"key":"probe","label":"Linear probe","blurb":"Encoder frozen at its pretrained weights; only a linear head is trained."},{"key":"ft","label":"Fine-tune","blurb":"Everything trained end to end from a randomly initialised head."},{"key":"lpft","label":"LP-FT","blurb":"Probe first, then fine-tune everything starting from that head (Kumar et al., 2022)."}],"grid":[{"method":"tfidf","train":"imdb","eval":"imdb","accuracy":0.89,"auroc":0.95548,"oracle_accuracy":0.8925,"ece":0.04523,"positive_rate":0.5145,"n":4000},{"method":"tfidf","train":"imdb","eval":"sst2","accuracy":0.75584,"auroc":0.83595,"oracle_accuracy":0.76168,"ece":0.05019,"positive_rate":0.56425,"n":856},{"method":"tfidf","train":"imdb","eval":"yelp","accuracy":0.83225,"auroc":0.9174,"oracle_accuracy":0.83475,"ece":0.04099,"positive_rate":0.50475,"n":4000},{"method":"tfidf","train":"imdb","eval":"amazon","accuracy":0.815,"auroc":0.90786,"oracle_accuracy":0.8245,"ece":0.03108,"positive_rate":0.541,"n":4000},{"method":"probe","train":"imdb","eval":"imdb","accuracy":0.85325,"auroc":0.92651,"oracle_accuracy":0.854,"ece":0.01842,"positive_rate":0.49425,"n":4000},{"method":"probe","train":"imdb","eval":"sst2","accuracy":0.81659,"auroc":0.89228,"oracle_accuracy":0.82944,"ece":0.08696,"positive_rate":0.54556,"n":856},{"method":"probe","train":"imdb","eval":"yelp","accuracy":0.776,"auroc":0.92162,"oracle_accuracy":0.848,"ece":0.06786,"positive_rate":0.6575,"n":4000},{"method":"probe","train":"imdb","eval":"amazon","accuracy":0.8245,"auroc":0.92313,"oracle_accuracy":0.8445,"ece":0.03808,"positive_rate":0.572,"n":4000},{"method":"tfidf","train":"sst2","eval":"imdb","accuracy":0.74075,"auroc":0.87223,"oracle_accuracy":0.797,"ece":0.03908,"positive_rate":0.31825,"n":4000},{"method":"tfidf","train":"sst2","eval":"sst2","accuracy":0.75935,"auroc":0.85429,"oracle_accuracy":0.77453,"ece":0.04403,"positive_rate":0.51402,"n":856},{"method":"tfidf","train":"sst2","eval":"yelp","accuracy":0.744,"auroc":0.85866,"oracle_accuracy":0.77525,"ece":0.02788,"positive_rate":0.3205,"n":4000},{"method":"tfidf","train":"sst2","eval":"amazon","accuracy":0.7315,"auroc":0.84198,"oracle_accuracy":0.766,"ece":0.03088,"positive_rate":0.344,"n":4000},{"method":"probe","train":"sst2","eval":"imdb","accuracy":0.803,"auroc":0.89812,"oracle_accuracy":0.81725,"ece":0.04895,"positive_rate":0.3975,"n":4000},{"method":"probe","train":"sst2","eval":"sst2","accuracy":0.84112,"auroc":0.91053,"oracle_accuracy":0.84112,"ece":0.02411,"positive_rate":0.48364,"n":856},{"method":"probe","train":"sst2","eval":"yelp","accuracy":0.85325,"auroc":0.94071,"oracle_accuracy":0.868,"ece":0.05162,"positive_rate":0.42675,"n":4000},{"method":"probe","train":"sst2","eval":"amazon","accuracy":0.8165,"auroc":0.91471,"oracle_accuracy":0.83075,"ece":0.01821,"positive_rate":0.3815,"n":4000},{"method":"tfidf","train":"yelp","eval":"imdb","accuracy":0.77325,"auroc":0.88132,"oracle_accuracy":0.79675,"ece":0.05488,"positive_rate":0.38825,"n":4000},{"method":"tfidf","train":"yelp","eval":"sst2","accuracy":0.68575,"auroc":0.75413,"oracle_accuracy":0.69393,"ece":0.03664,"positive_rate":0.58294,"n":856},{"method":"tfidf","train":"yelp","eval":"yelp","accuracy":0.9145,"auroc":0.97352,"oracle_accuracy":0.9165,"ece":0.0887,"positive_rate":0.496,"n":4000},{"method":"tfidf","train":"yelp","eval":"amazon","accuracy":0.8345,"auroc":0.91429,"oracle_accuracy":0.83775,"ece":0.0922,"positive_rate":0.4695,"n":4000},{"method":"probe","train":"yelp","eval":"imdb","accuracy":0.775,"auroc":0.85999,"oracle_accuracy":0.779,"ece":0.0705,"positive_rate":0.4375,"n":4000},{"method":"probe","train":"yelp","eval":"sst2","accuracy":0.76519,"auroc":0.84705,"oracle_accuracy":0.77103,"ece":0.12793,"positive_rate":0.51986,"n":856},{"method":"probe","train":"yelp","eval":"yelp","accuracy":0.90625,"auroc":0.9663,"oracle_accuracy":0.90625,"ece":0.01732,"positive_rate":0.48925,"n":4000},{"method":"probe","train":"yelp","eval":"amazon","accuracy":0.84625,"auroc":0.93124,"oracle_accuracy":0.8565,"ece":0.03564,"positive_rate":0.52975,"n":4000},{"method":"tfidf","train":"amazon","eval":"imdb","accuracy":0.8315,"auroc":0.91761,"oracle_accuracy":0.836,"ece":0.08032,"positive_rate":0.466,"n":4000},{"method":"tfidf","train":"amazon","eval":"sst2","accuracy":0.71612,"auroc":0.79321,"oracle_accuracy":0.72196,"ece":0.04625,"positive_rate":0.57126,"n":856},{"method":"tfidf","train":"amazon","eval":"yelp","accuracy":0.865,"auroc":0.94176,"oracle_accuracy":0.86775,"ece":0.09825,"positive_rate":0.495,"n":4000},{"method":"tfidf","train":"amazon","eval":"amazon","accuracy":0.87675,"auroc":0.94913,"oracle_accuracy":0.88,"ece":0.09483,"positive_rate":0.49475,"n":4000},{"method":"probe","train":"amazon","eval":"imdb","accuracy":0.82,"auroc":0.90316,"oracle_accuracy":0.82275,"ece":0.01495,"positive_rate":0.47,"n":4000},{"method":"probe","train":"amazon","eval":"sst2","accuracy":0.79089,"auroc":0.88588,"oracle_accuracy":0.81659,"ece":0.08982,"positive_rate":0.55257,"n":856},{"method":"probe","train":"amazon","eval":"yelp","accuracy":0.86275,"auroc":0.94988,"oracle_accuracy":0.87675,"ece":0.01488,"positive_rate":0.55825,"n":4000},{"method":"probe","train":"amazon","eval":"amazon","accuracy":0.88425,"auroc":0.95476,"oracle_accuracy":0.88725,"ece":0.01302,"positive_rate":0.48875,"n":4000}],"report":{"methods":[{"method":"tfidf","id_accuracy":0.8601489485981308,"ood_accuracy":0.7771216900311527,"ood_gap":0.08302725856697812,"id_auroc":0.9331054482869683,"ood_auroc":0.8697007361605453,"ood_oracle_accuracy":0.7927766744548287,"id_ece":0.06819728512338892,"ood_ece":0.05238538153535782,"total_drop":0.08302725856697812,"ranking_drop":0.06737227414330216,"placement_drop":0.015654984423675966},{"method":"probe","id_accuracy":0.8712178738317757,"ood_accuracy":0.8124927959501558,"ood_gap":0.05872507788161996,"id_auroc":0.9395254573161411,"ood_auroc":0.9056478458908347,"ood_oracle_accuracy":0.830046339563863,"id_ece":0.018215950827522174,"ood_ece":0.0554498978555863,"total_drop":0.05872507788161996,"ranking_drop":0.04117153426791276,"placement_drop":0.0175535436137072}],"headline":{"primary_method":"probe","tfidf_id_accuracy":0.8601489485981308,"tfidf_ood_accuracy":0.7771216900311527,"tfidf_ood_gap":0.08302725856697812,"tfidf_ranking_drop":0.06737227414330216,"tfidf_placement_drop":0.015654984423675966,"tfidf_placement_share":0.1885523464688056,"probe_id_accuracy":0.8712178738317757,"probe_ood_accuracy":0.8124927959501558,"probe_ood_gap":0.05872507788161996,"probe_ranking_drop":0.04117153426791276,"probe_placement_drop":0.0175535436137072,"probe_placement_share":0.2989105208015601},"shift_axes":[{"method":"probe","shift":"topic-different/length-different","accuracy":0.806456191588785,"n":4},{"method":"probe","shift":"topic-different/length-same","accuracy":0.798875,"n":4},{"method":"probe","shift":"topic-same/length-different","accuracy":0.8097943925233645,"n":2},{"method":"probe","shift":"topic-same/length-same","accuracy":0.8545,"n":2},{"method":"tfidf","shift":"topic-different/length-different","accuracy":0.7193422897196262,"n":4},{"method":"tfidf","shift":"topic-different/length-same","accuracy":0.813,"n":4},{"method":"tfidf","shift":"topic-same/length-different","accuracy":0.7482955607476636,"n":2},{"method":"tfidf","shift":"topic-same/length-same","accuracy":0.84975,"n":2}],"mechanism":[{"method":"probe","scope":"id","cka_vs_pretrained":1.0,"posthoc_probe_accuracy":null,"pretrained_probe_accuracy":null,"probe_accuracy_delta":0.0,"n":4},{"method":"probe","scope":"ood","cka_vs_pretrained":1.0,"posthoc_probe_accuracy":null,"pretrained_probe_accuracy":null,"probe_accuracy_delta":0.0,"n":12}],"selection_regret":{"rows":[{"train_domain":"imdb","chosen_on_id":"tfidf","chosen_id_accuracy":0.89,"chosen_ood_accuracy":0.8010303738317757,"best_ood_method":"probe","best_ood_accuracy":0.8056962616822431,"regret":0.004665887850467398,"agrees":false},{"train_domain":"sst2","chosen_on_id":"probe","chosen_id_accuracy":0.8411214953271028,"chosen_ood_accuracy":0.82425,"best_ood_method":"probe","best_ood_accuracy":0.82425,"regret":0.0,"agrees":true},{"train_domain":"yelp","chosen_on_id":"tfidf","chosen_id_accuracy":0.9145,"chosen_ood_accuracy":0.7644992211838005,"best_ood_method":"probe","best_ood_accuracy":0.7954789719626169,"regret":0.030979750778816317,"agrees":false},{"train_domain":"amazon","chosen_on_id":"probe","chosen_id_accuracy":0.88425,"chosen_ood_accuracy":0.8245459501557632,"best_ood_method":"probe","best_ood_accuracy":0.8245459501557632,"regret":0.0,"agrees":true}],"disagreements":2,"mean_regret":0.008911409657320929,"max_regret":0.030979750778816317},"checkpoint_regret":{"rows":[],"mean_regret":null,"runs_where_earlier_checkpoint_was_better":0,"runs":0},"ft_vs_lpft":{"baseline":"ft","challenger":"lpft","id_mean_delta":null,"ood_mean_delta":null,"ood_cells":0,"ood_wins":0,"cells":[]},"probe_vs_ft":{"baseline":"probe","challenger":"ft","id_mean_delta":null,"ood_mean_delta":null,"ood_cells":0,"ood_wins":0,"cells":[]},"tfidf_vs_ft":{"baseline":"tfidf","challenger":"ft","id_mean_delta":null,"ood_mean_delta":null,"ood_cells":0,"ood_wins":0,"cells":[]}},"examples":[{"label":"Restaurant","text":"The service was slow but the pasta was worth the wait."},{"label":"Product","text":"Battery lasted about two days on a charge and the case feels solid after a month of being thrown in a bag. Setup took under five minutes."},{"label":"Film","text":"The premise is wasted on a script that explains every feeling out loud, and even the cast look bored by the third act. A long two hours."},{"label":"One-liner","text":"a quietly devastating performance that lingers long after the credits"}]} |