Update to smaller bert model, improves performance a lot

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "google/bert_uncased_L-8_H-256_A-4",
+  "_name_or_path": "google/bert_uncased_L-8_H-128_A-2",
   "architectures": [
     "Hydra"
   ],
@@ -7,173 +7,367 @@
   "classifier_dropout": null,
   "hidden_act": "gelu",
   "hidden_dropout_prob": 0.1,
-  "hidden_size": 256,
+  "hidden_size": 128,
   "id2label": null,
   "initializer_range": 0.02,
-  "intermediate_size": 1024,
+  "intermediate_size": 512,
   "label2id": null,
   "label_groups": [
     {
-      "amenity=arts_centre": 0,
-      "amenity=atm": 1,
-      "amenity=bank": 2,
-      "amenity=bar": 3,
-      "amenity=bench": 4,
-      "amenity=bicycle_rental": 5,
-      "amenity=bus_station": 6,
-      "amenity=cafe": 7,
-      "amenity=camping": 8,
-      "amenity=car_rental": 9,
-      "amenity=car_wash": 10,
-      "amenity=casino": 11,
-      "amenity=charging_station": 12,
-      "amenity=childcare": 13,
-      "amenity=cinema": 14,
-      "amenity=clinic": 15,
-      "amenity=college": 16,
-      "amenity=community_centre": 17,
-      "amenity=courthouse": 18,
-      "amenity=dentist": 19,
-      "amenity=doctors": 20,
-      "amenity=dormitory": 21,
-      "amenity=events_venue": 22,
-      "amenity=fast_food": 23,
-      "amenity=fire_station": 24,
-      "amenity=food_court": 25,
-      "amenity=fountain": 26,
-      "amenity=fuel": 27,
-      "amenity=grave_yard": 28,
-      "amenity=hospital": 29,
-      "amenity=ice_cream": 30,
-      "amenity=kindergarten": 31,
-      "amenity=library": 32,
-      "amenity=marketplace": 33,
-      "amenity=meditation_centre": 34,
-      "amenity=monastery": 35,
-      "amenity=nightclub": 36,
-      "amenity=parking": 37,
-      "amenity=pharmacy": 38,
-      "amenity=place_of_worship": 39,
-      "amenity=police": 40,
-      "amenity=post_depot": 41,
-      "amenity=post_office": 42,
-      "amenity=prison": 43,
-      "amenity=pub": 44,
-      "amenity=ranger_station": 45,
-      "amenity=recycling": 46,
-      "amenity=research_institute": 47,
-      "amenity=restaurant": 48,
-      "amenity=school": 49,
-      "amenity=shelter": 50,
-      "amenity=shower": 51,
-      "amenity=social_centre": 52,
-      "amenity=social_facility": 53,
-      "amenity=spa": 54,
-      "amenity=studio": 55,
-      "amenity=theatre": 56,
-      "amenity=toilets": 57,
-      "amenity=townhall": 58,
-      "amenity=university": 59,
-      "amenity=veterinary": 60,
-      "amenity=waste_transfer_station": 61,
-      "amenity=watering_place": 62,
-      "healthcare=alternative": 63,
-      "healthcare=blood_donation": 64,
-      "healthcare=clinic": 65,
-      "healthcare=dentist": 66,
-      "healthcare=hospice": 67,
-      "healthcare=laboratory": 68,
-      "healthcare=physiotherapist": 69,
-      "healthcare=rehabilitation": 70,
-      "leisure=beach_resort": 71,
-      "leisure=bowling_alley": 72,
-      "leisure=dance": 73,
-      "leisure=dog_park": 74,
-      "leisure=fitness_centre": 75,
-      "leisure=garden": 76,
-      "leisure=golf_course": 77,
-      "leisure=marina": 78,
-      "leisure=nature_reserve": 79,
-      "leisure=outdoor_seating": 80,
-      "leisure=park": 81,
-      "leisure=pitch": 82,
-      "leisure=playground": 83,
-      "leisure=recreation_ground": 84,
-      "leisure=resort": 85,
-      "leisure=sports_centre": 86,
-      "leisure=sports_hall": 87,
-      "leisure=stadium": 88,
-      "leisure=swimming_pool": 89,
-      "leisure=yes": 90,
-      "natural=beach": 91,
-      "natural=coastline": 92,
-      "natural=grassland": 93,
-      "natural=reef": 94,
-      "natural=water": 95,
-      "natural=wetland": 96,
-      "natural=wood": 97,
-      "shop=agrarian": 98,
-      "shop=alcohol": 99,
-      "shop=antiques": 100,
-      "shop=art": 101,
-      "shop=auction_house": 102,
-      "shop=baby_goods": 103,
-      "shop=bakery": 104,
-      "shop=beauty": 105,
-      "shop=bicycle": 106,
-      "shop=books": 107,
-      "shop=cannabis": 108,
-      "shop=car": 109,
-      "shop=car_parts": 110,
-      "shop=car_repair": 111,
-      "shop=charity": 112,
-      "shop=chemist": 113,
-      "shop=clothes": 114,
-      "shop=coffee": 115,
-      "shop=collector": 116,
-      "shop=confectionery": 117,
-      "shop=convenience": 118,
-      "shop=copyshop": 119,
-      "shop=country_store": 120,
-      "shop=department_store": 121,
-      "shop=doityourself": 122,
-      "shop=dry_cleaning": 123,
-      "shop=electronics": 124,
-      "shop=fireplace": 125,
-      "shop=flooring": 126,
-      "shop=furniture": 127,
-      "shop=garden_centre": 128,
-      "shop=gift": 129,
-      "shop=hairdresser": 130,
-      "shop=interior_decoration": 131,
-      "shop=jewelry": 132,
-      "shop=laundry": 133,
-      "shop=mall": 134,
-      "shop=motorcycle": 135,
-      "shop=nutrition_supplements": 136,
-      "shop=optician": 137,
-      "shop=outdoor": 138,
-      "shop=paint": 139,
-      "shop=pastry": 140,
-      "shop=second_hand": 141,
-      "shop=shoes": 142,
-      "shop=sports": 143,
-      "shop=storage_rental": 144,
-      "shop=supermarket": 145,
-      "shop=tailor": 146,
-      "shop=telescopes": 147,
-      "shop=trade": 148,
-      "shop=trailer": 149,
-      "shop=truck_repair": 150,
-      "shop=variety_store": 151,
-      "shop=wholesale": 152,
-      "shop=yes": 153
+      "amenity=animal_boarding": 0,
+      "amenity=animal_shelter": 1,
+      "amenity=archive": 2,
+      "amenity=arts_centre": 3,
+      "amenity=atm": 4,
+      "amenity=bakery": 5,
+      "amenity=bank": 6,
+      "amenity=bar": 7,
+      "amenity=beach_resort": 8,
+      "amenity=bench": 9,
+      "amenity=bicycle_parking": 10,
+      "amenity=bicycle_rental": 11,
+      "amenity=biergarten": 12,
+      "amenity=boat_rental": 13,
+      "amenity=brewery": 14,
+      "amenity=bus_station": 15,
+      "amenity=cafe": 16,
+      "amenity=camping": 17,
+      "amenity=canteen": 18,
+      "amenity=car_pooling": 19,
+      "amenity=car_rental": 20,
+      "amenity=car_wash": 21,
+      "amenity=casino": 22,
+      "amenity=charging_station": 23,
+      "amenity=childcare": 24,
+      "amenity=cinema": 25,
+      "amenity=clinic": 26,
+      "amenity=clock": 27,
+      "amenity=coast_guard": 28,
+      "amenity=college": 29,
+      "amenity=community_centre": 30,
+      "amenity=conference_centre": 31,
+      "amenity=courthouse": 32,
+      "amenity=credit union": 33,
+      "amenity=dentist": 34,
+      "amenity=doctors": 35,
+      "amenity=dojo": 36,
+      "amenity=dormitory": 37,
+      "amenity=dressing_room": 38,
+      "amenity=event_hall": 39,
+      "amenity=events_venue": 40,
+      "amenity=fast_food": 41,
+      "amenity=ferry_terminal": 42,
+      "amenity=festival_grounds": 43,
+      "amenity=fire_station": 44,
+      "amenity=first_aid_school": 45,
+      "amenity=fish_hatchery": 46,
+      "amenity=food_court": 47,
+      "amenity=fortune_teller": 48,
+      "amenity=fountain": 49,
+      "amenity=fuel": 50,
+      "amenity=grave_yard": 51,
+      "amenity=grocery store": 52,
+      "amenity=healthcare": 53,
+      "amenity=hospital": 54,
+      "amenity=ice_cream": 55,
+      "amenity=indoor_range": 56,
+      "amenity=kindergarten": 57,
+      "amenity=library": 58,
+      "amenity=lounge": 59,
+      "amenity=love_hotel": 60,
+      "amenity=marketplace": 61,
+      "amenity=meditation_centre": 62,
+      "amenity=monastery": 63,
+      "amenity=mortuary": 64,
+      "amenity=motorcycle_parking": 65,
+      "amenity=music_school": 66,
+      "amenity=music_venue": 67,
+      "amenity=nightclub": 68,
+      "amenity=nursing_home": 69,
+      "amenity=office": 70,
+      "amenity=parking": 71,
+      "amenity=parking_space": 72,
+      "amenity=pharmacy": 73,
+      "amenity=place_of_worship": 74,
+      "amenity=place_of_worship;grave_yard": 75,
+      "amenity=police": 76,
+      "amenity=post_box": 77,
+      "amenity=post_depot": 78,
+      "amenity=post_office": 79,
+      "amenity=prison": 80,
+      "amenity=pub": 81,
+      "amenity=public_building": 82,
+      "amenity=public_facility": 83,
+      "amenity=ranger_station": 84,
+      "amenity=reception_desk": 85,
+      "amenity=recycling": 86,
+      "amenity=research_institute": 87,
+      "amenity=resort": 88,
+      "amenity=restaurant": 89,
+      "amenity=school": 90,
+      "amenity=shelter": 91,
+      "amenity=shower": 92,
+      "amenity=social_centre": 93,
+      "amenity=social_facility": 94,
+      "amenity=spa": 95,
+      "amenity=stripclub": 96,
+      "amenity=studio": 97,
+      "amenity=supermarket": 98,
+      "amenity=theatre": 99,
+      "amenity=toilets": 100,
+      "amenity=townhall": 101,
+      "amenity=training": 102,
+      "amenity=university": 103,
+      "amenity=vehicle_inspection": 104,
+      "amenity=veterinary": 105,
+      "amenity=waste_transfer_station": 106,
+      "amenity=watering_place": 107,
+      "amenity=yacht_club": 108,
+      "healthcare=alternative": 109,
+      "healthcare=blood_donation": 110,
+      "healthcare=centre": 111,
+      "healthcare=clinic": 112,
+      "healthcare=counselling": 113,
+      "healthcare=dentist": 114,
+      "healthcare=doctor": 115,
+      "healthcare=hospice": 116,
+      "healthcare=hospital": 117,
+      "healthcare=laboratory": 118,
+      "healthcare=optometrist": 119,
+      "healthcare=pharmacy": 120,
+      "healthcare=physiotherapist": 121,
+      "healthcare=psychotherapist": 122,
+      "healthcare=rehabilitation": 123,
+      "healthcare=testing_site": 124,
+      "healthcare=yes": 125,
+      "landuse=allotments": 126,
+      "landuse=aquaculture": 127,
+      "landuse=brownfield": 128,
+      "landuse=cemetery": 129,
+      "landuse=civic": 130,
+      "landuse=commercial": 131,
+      "landuse=conservation": 132,
+      "landuse=construction": 133,
+      "landuse=depot": 134,
+      "landuse=education": 135,
+      "landuse=fairground": 136,
+      "landuse=farmland": 137,
+      "landuse=farmyard": 138,
+      "landuse=forest": 139,
+      "landuse=garages": 140,
+      "landuse=grass": 141,
+      "landuse=greenfield": 142,
+      "landuse=greenhouse_horticulture": 143,
+      "landuse=industrial": 144,
+      "landuse=landfill": 145,
+      "landuse=military": 146,
+      "landuse=orchard": 147,
+      "landuse=public": 148,
+      "landuse=quarry": 149,
+      "landuse=recreation_ground": 150,
+      "landuse=religious": 151,
+      "landuse=reservoir": 152,
+      "landuse=reservoir_watershed": 153,
+      "landuse=residential": 154,
+      "landuse=retail": 155,
+      "landuse=tourism": 156,
+      "landuse=village_green": 157,
+      "landuse=vineyard": 158,
+      "landuse=watershed_protection": 159,
+      "landuse=winter_sports": 160,
+      "leisure=amusement_arcade": 161,
+      "leisure=bandstand": 162,
+      "leisure=beach_resort": 163,
+      "leisure=bowling_alley": 164,
+      "leisure=dance": 165,
+      "leisure=disc_golf_course": 166,
+      "leisure=dog_park": 167,
+      "leisure=escape_game": 168,
+      "leisure=fishing": 169,
+      "leisure=fitness_centre": 170,
+      "leisure=garden": 171,
+      "leisure=golf_course": 172,
+      "leisure=hackerspace": 173,
+      "leisure=ice_rink": 174,
+      "leisure=marina": 175,
+      "leisure=miniature_golf": 176,
+      "leisure=nature_reserve": 177,
+      "leisure=outdoor_seating": 178,
+      "leisure=park": 179,
+      "leisure=park;outdoor_seating": 180,
+      "leisure=pitch": 181,
+      "leisure=playground": 182,
+      "leisure=recreation_ground": 183,
+      "leisure=resort": 184,
+      "leisure=sports_centre": 185,
+      "leisure=sports_hall": 186,
+      "leisure=stadium": 187,
+      "leisure=summer_camp": 188,
+      "leisure=swimming_pool": 189,
+      "leisure=tanning_salon": 190,
+      "leisure=track": 191,
+      "leisure=trampoline_park": 192,
+      "leisure=water_park": 193,
+      "leisure=yes": 194,
+      "natural=bay": 195,
+      "natural=beach": 196,
+      "natural=cliff": 197,
+      "natural=coastline": 198,
+      "natural=glacier": 199,
+      "natural=grassland": 200,
+      "natural=reef": 201,
+      "natural=scree": 202,
+      "natural=stone": 203,
+      "natural=strait": 204,
+      "natural=water": 205,
+      "natural=wetland": 206,
+      "natural=wood": 207,
+      "shop=agrarian": 208,
+      "shop=alcohol": 209,
+      "shop=anime": 210,
+      "shop=antiques": 211,
+      "shop=appliance": 212,
+      "shop=art": 213,
+      "shop=auction_house": 214,
+      "shop=baby_goods": 215,
+      "shop=bag": 216,
+      "shop=bakery": 217,
+      "shop=bathroom_furnishing": 218,
+      "shop=beauty": 219,
+      "shop=bed": 220,
+      "shop=bicycle": 221,
+      "shop=boat": 222,
+      "shop=books": 223,
+      "shop=butcher": 224,
+      "shop=cannabis": 225,
+      "shop=car": 226,
+      "shop=car_parts": 227,
+      "shop=car_repair": 228,
+      "shop=caravan": 229,
+      "shop=carpet": 230,
+      "shop=charity": 231,
+      "shop=cheese": 232,
+      "shop=chemist": 233,
+      "shop=clothes": 234,
+      "shop=coffee": 235,
+      "shop=collector": 236,
+      "shop=computer": 237,
+      "shop=confectionery": 238,
+      "shop=convenience": 239,
+      "shop=copyshop": 240,
+      "shop=cosmetics": 241,
+      "shop=country_store": 242,
+      "shop=craft": 243,
+      "shop=dairy": 244,
+      "shop=deli": 245,
+      "shop=dentures": 246,
+      "shop=department_store": 247,
+      "shop=doityourself": 248,
+      "shop=dry_cleaning": 249,
+      "shop=e-cigarette": 250,
+      "shop=electronics": 251,
+      "shop=equipment_rental": 252,
+      "shop=erotic": 253,
+      "shop=estate_agent": 254,
+      "shop=fabric": 255,
+      "shop=farm": 256,
+      "shop=fashion_accessories": 257,
+      "shop=fireplace": 258,
+      "shop=fishing": 259,
+      "shop=flooring": 260,
+      "shop=florist": 261,
+      "shop=frame": 262,
+      "shop=funeral_directors": 263,
+      "shop=furniture": 264,
+      "shop=games": 265,
+      "shop=garden_centre": 266,
+      "shop=gas": 267,
+      "shop=general": 268,
+      "shop=generators": 269,
+      "shop=gift": 270,
+      "shop=greengrocer": 271,
+      "shop=grocery": 272,
+      "shop=hairdresser": 273,
+      "shop=hairdresser_supply": 274,
+      "shop=hardware": 275,
+      "shop=health_food": 276,
+      "shop=hearing_aids": 277,
+      "shop=household_linen": 278,
+      "shop=houseware": 279,
+      "shop=hunting": 280,
+      "shop=implement": 281,
+      "shop=interior_decoration": 282,
+      "shop=internet_service": 283,
+      "shop=jewelry": 284,
+      "shop=laundry": 285,
+      "shop=lawyer": 286,
+      "shop=lighting": 287,
+      "shop=loaning_agency": 288,
+      "shop=locksmith": 289,
+      "shop=mall": 290,
+      "shop=massage": 291,
+      "shop=medical_supply": 292,
+      "shop=mobile_phone": 293,
+      "shop=model": 294,
+      "shop=money_lender": 295,
+      "shop=motorcycle": 296,
+      "shop=motorcycle_repair": 297,
+      "shop=motorsports": 298,
+      "shop=museum_store": 299,
+      "shop=music": 300,
+      "shop=musical_instrument": 301,
+      "shop=nutrition_supplements": 302,
+      "shop=office_supplies": 303,
+      "shop=optician": 304,
+      "shop=outdoor": 305,
+      "shop=paint": 306,
+      "shop=party": 307,
+      "shop=pastry": 308,
+      "shop=pawnbroker": 309,
+      "shop=pest_control": 310,
+      "shop=pet": 311,
+      "shop=pet_grooming": 312,
+      "shop=photo": 313,
+      "shop=pottery": 314,
+      "shop=printer_ink": 315,
+      "shop=pyrotechnics": 316,
+      "shop=religion": 317,
+      "shop=rental": 318,
+      "shop=repair": 319,
+      "shop=seafood": 320,
+      "shop=second_hand": 321,
+      "shop=shoes": 322,
+      "shop=sports": 323,
+      "shop=stationery": 324,
+      "shop=storage_rental": 325,
+      "shop=supermarket": 326,
+      "shop=swimming_pool": 327,
+      "shop=tailor": 328,
+      "shop=tattoo": 329,
+      "shop=telecommunication": 330,
+      "shop=telescopes": 331,
+      "shop=tobacco": 332,
+      "shop=toys": 333,
+      "shop=trade": 334,
+      "shop=trailer": 335,
+      "shop=travel_agency": 336,
+      "shop=trophy": 337,
+      "shop=truck_repair": 338,
+      "shop=tyres": 339,
+      "shop=variety_store": 340,
+      "shop=water": 341,
+      "shop=weapons": 342,
+      "shop=wholesale": 343,
+      "shop=wigs": 344,
+      "shop=wine": 345,
+      "shop=wine;convenience": 346,
+      "shop=yes": 347
     }
   ],
   "layer_norm_eps": 1e-12,
   "max_position_embeddings": 512,
-  "model_type": "hydra",
-  "num_attention_heads": 4,
+  "model_type": "bert",
+  "num_attention_heads": 2,
   "num_hidden_layers": 8,
   "num_labels": 2,
   "pad_token_id": 0,

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d733f3a39209f000904e4f69d651d284085a9b5ebbce7c19614c16f2c6cdebde
-size 57755096
+oid sha256:60c9523925a4ac0ae2f5c60d8e81dec6019556a03bebd8ce760fe29208c14656
+size 22563016

tokenizer_config.json

@@ -46,7 +46,7 @@
   "do_basic_tokenize": true,
   "do_lower_case": true,
   "mask_token": "[MASK]",
-  "max_length": 64,
+  "max_length": 32,
   "model_max_length": 1000000000000000019884624838656,
   "never_split": null,
   "pad_token": "[PAD]",

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b8a92453bb254c1a201f7ae6e35d25baf657dbc8af553870150deb1d4b6fb2bb
+oid sha256:1664af370cef66ec7f3efc33bb5cc669dcca19da7b1e173b4e1f4a5395e4cb7e
 size 4856