Upload 6 files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

recipe.yaml

@@ -0,0 +1,12 @@
+default_stage:
+  default_modifiers:
+    GPTQModifier:
+      targets: [Linear]
+      ignore: ['re:.*lm_head$', 're:.*qkv$', 're:.*fc1$', 're:.*fc2$', 're:.*attn.proj$',
+        're:.*up_proj$', 're:.*gate_proj$', 're:.*down_proj$', 're:.*router$', 're:.*self_attn.get_query_key*',
+        're:.*self_attn.o_proj*', 're:.*self_attn.v_proj*']
+      scheme: W4A16
+      sequential_update: true
+      block_size: 128
+      dampening_frac: 0.01
+      offload_hessians: false

special_tokens_map.json

@@ -0,0 +1,1092 @@
+{
+  "additional_special_tokens": [
+    "<s>",
+    "<eod>",
+    "<unk>",
+    "<sep>",
+    "<pad>",
+    "<mask>",
+    "<predict>",
+    "<FIM_SUFFIX>",
+    "<FIM_PREFIX>",
+    "<FIM_MIDDLE>",
+    "<commit_before>",
+    "<commit_msg>",
+    "<commit_after>",
+    "<jupyter_start>",
+    "<jupyter_text>",
+    "<jupyter_code>",
+    "<jupyter_output>",
+    "<empty_output>",
+    "<repo_name>",
+    "<file_sep>",
+    "<BOS>",
+    "<IMAGE>",
+    "</IMAGE>",
+    "<grounding>",
+    "<obj>",
+    "</obj>",
+    "<box>",
+    "</box>",
+    "<point>",
+    "</point>",
+    "<3dbox>",
+    "</3dbox>",
+    "<depth>",
+    "</depth>",
+    "s000",
+    "s001",
+    "s002",
+    "s003",
+    "s004",
+    "s005",
+    "s006",
+    "s007",
+    "s008",
+    "s009",
+    "s010",
+    "s011",
+    "s012",
+    "s013",
+    "s014",
+    "s015",
+    "s016",
+    "s017",
+    "s018",
+    "s019",
+    "s020",
+    "s021",
+    "s022",
+    "s023",
+    "s024",
+    "s025",
+    "s026",
+    "s027",
+    "s028",
+    "s029",
+    "s030",
+    "s031",
+    "s032",
+    "s033",
+    "s034",
+    "s035",
+    "s036",
+    "s037",
+    "s038",
+    "s039",
+    "s040",
+    "s041",
+    "s042",
+    "s043",
+    "s044",
+    "s045",
+    "s046",
+    "s047",
+    "s048",
+    "s049",
+    "s050",
+    "s051",
+    "s052",
+    "s053",
+    "s054",
+    "s055",
+    "s056",
+    "s057",
+    "s058",
+    "s059",
+    "s060",
+    "s061",
+    "s062",
+    "s063",
+    "s064",
+    "s065",
+    "s066",
+    "s067",
+    "s068",
+    "s069",
+    "s070",
+    "s071",
+    "s072",
+    "s073",
+    "s074",
+    "s075",
+    "s076",
+    "s077",
+    "s078",
+    "s079",
+    "s080",
+    "s081",
+    "s082",
+    "s083",
+    "s084",
+    "s085",
+    "s086",
+    "s087",
+    "s088",
+    "s089",
+    "s090",
+    "s091",
+    "s092",
+    "s093",
+    "s094",
+    "s095",
+    "s096",
+    "s097",
+    "s098",
+    "s099",
+    "s100",
+    "s101",
+    "s102",
+    "s103",
+    "s104",
+    "s105",
+    "s106",
+    "s107",
+    "s108",
+    "s109",
+    "s110",
+    "s111",
+    "s112",
+    "s113",
+    "s114",
+    "s115",
+    "s116",
+    "s117",
+    "s118",
+    "s119",
+    "s120",
+    "s121",
+    "s122",
+    "s123",
+    "s124",
+    "s125",
+    "s126",
+    "s127",
+    "s128",
+    "s129",
+    "s130",
+    "s131",
+    "s132",
+    "s133",
+    "s134",
+    "s135",
+    "s136",
+    "s137",
+    "s138",
+    "s139",
+    "s140",
+    "s141",
+    "s142",
+    "s143",
+    "s144",
+    "s145",
+    "s146",
+    "s147",
+    "s148",
+    "s149",
+    "s150",
+    "s151",
+    "s152",
+    "s153",
+    "s154",
+    "s155",
+    "s156",
+    "s157",
+    "s158",
+    "s159",
+    "s160",
+    "s161",
+    "s162",
+    "s163",
+    "s164",
+    "s165",
+    "s166",
+    "s167",
+    "s168",
+    "s169",
+    "s170",
+    "s171",
+    "s172",
+    "s173",
+    "s174",
+    "s175",
+    "s176",
+    "s177",
+    "s178",
+    "s179",
+    "s180",
+    "s181",
+    "s182",
+    "s183",
+    "s184",
+    "s185",
+    "s186",
+    "s187",
+    "s188",
+    "s189",
+    "s190",
+    "s191",
+    "s192",
+    "s193",
+    "s194",
+    "s195",
+    "s196",
+    "s197",
+    "s198",
+    "s199",
+    "s200",
+    "s201",
+    "s202",
+    "s203",
+    "s204",
+    "s205",
+    "s206",
+    "s207",
+    "s208",
+    "s209",
+    "s210",
+    "s211",
+    "s212",
+    "s213",
+    "s214",
+    "s215",
+    "s216",
+    "s217",
+    "s218",
+    "s219",
+    "s220",
+    "s221",
+    "s222",
+    "s223",
+    "s224",
+    "s225",
+    "s226",
+    "s227",
+    "s228",
+    "s229",
+    "s230",
+    "s231",
+    "s232",
+    "s233",
+    "s234",
+    "s235",
+    "s236",
+    "s237",
+    "s238",
+    "s239",
+    "s240",
+    "s241",
+    "s242",
+    "s243",
+    "s244",
+    "s245",
+    "s246",
+    "s247",
+    "s248",
+    "s249",
+    "s250",
+    "s251",
+    "s252",
+    "s253",
+    "s254",
+    "s255",
+    "s256",
+    "s257",
+    "s258",
+    "s259",
+    "s260",
+    "s261",
+    "s262",
+    "s263",
+    "s264",
+    "s265",
+    "s266",
+    "s267",
+    "s268",
+    "s269",
+    "s270",
+    "s271",
+    "s272",
+    "s273",
+    "s274",
+    "s275",
+    "s276",
+    "s277",
+    "s278",
+    "s279",
+    "s280",
+    "s281",
+    "s282",
+    "s283",
+    "s284",
+    "s285",
+    "s286",
+    "s287",
+    "s288",
+    "s289",
+    "s290",
+    "s291",
+    "s292",
+    "s293",
+    "s294",
+    "s295",
+    "s296",
+    "s297",
+    "s298",
+    "s299",
+    "s300",
+    "s301",
+    "s302",
+    "s303",
+    "s304",
+    "s305",
+    "s306",
+    "s307",
+    "s308",
+    "s309",
+    "s310",
+    "s311",
+    "s312",
+    "s313",
+    "s314",
+    "s315",
+    "s316",
+    "s317",
+    "s318",
+    "s319",
+    "s320",
+    "s321",
+    "s322",
+    "s323",
+    "s324",
+    "s325",
+    "s326",
+    "s327",
+    "s328",
+    "s329",
+    "s330",
+    "s331",
+    "s332",
+    "s333",
+    "s334",
+    "s335",
+    "s336",
+    "s337",
+    "s338",
+    "s339",
+    "s340",
+    "s341",
+    "s342",
+    "s343",
+    "s344",
+    "s345",
+    "s346",
+    "s347",
+    "s348",
+    "s349",
+    "s350",
+    "s351",
+    "s352",
+    "s353",
+    "s354",
+    "s355",
+    "s356",
+    "s357",
+    "s358",
+    "s359",
+    "s360",
+    "s361",
+    "s362",
+    "s363",
+    "s364",
+    "s365",
+    "s366",
+    "s367",
+    "s368",
+    "s369",
+    "s370",
+    "s371",
+    "s372",
+    "s373",
+    "s374",
+    "s375",
+    "s376",
+    "s377",
+    "s378",
+    "s379",
+    "s380",
+    "s381",
+    "s382",
+    "s383",
+    "s384",
+    "s385",
+    "s386",
+    "s387",
+    "s388",
+    "s389",
+    "s390",
+    "s391",
+    "s392",
+    "s393",
+    "s394",
+    "s395",
+    "s396",
+    "s397",
+    "s398",
+    "s399",
+    "s400",
+    "s401",
+    "s402",
+    "s403",
+    "s404",
+    "s405",
+    "s406",
+    "s407",
+    "s408",
+    "s409",
+    "s410",
+    "s411",
+    "s412",
+    "s413",
+    "s414",
+    "s415",
+    "s416",
+    "s417",
+    "s418",
+    "s419",
+    "s420",
+    "s421",
+    "s422",
+    "s423",
+    "s424",
+    "s425",
+    "s426",
+    "s427",
+    "s428",
+    "s429",
+    "s430",
+    "s431",
+    "s432",
+    "s433",
+    "s434",
+    "s435",
+    "s436",
+    "s437",
+    "s438",
+    "s439",
+    "s440",
+    "s441",
+    "s442",
+    "s443",
+    "s444",
+    "s445",
+    "s446",
+    "s447",
+    "s448",
+    "s449",
+    "s450",
+    "s451",
+    "s452",
+    "s453",
+    "s454",
+    "s455",
+    "s456",
+    "s457",
+    "s458",
+    "s459",
+    "s460",
+    "s461",
+    "s462",
+    "s463",
+    "s464",
+    "s465",
+    "s466",
+    "s467",
+    "s468",
+    "s469",
+    "s470",
+    "s471",
+    "s472",
+    "s473",
+    "s474",
+    "s475",
+    "s476",
+    "s477",
+    "s478",
+    "s479",
+    "s480",
+    "s481",
+    "s482",
+    "s483",
+    "s484",
+    "s485",
+    "s486",
+    "s487",
+    "s488",
+    "s489",
+    "s490",
+    "s491",
+    "s492",
+    "s493",
+    "s494",
+    "s495",
+    "s496",
+    "s497",
+    "s498",
+    "s499",
+    "s500",
+    "s501",
+    "s502",
+    "s503",
+    "s504",
+    "s505",
+    "s506",
+    "s507",
+    "s508",
+    "s509",
+    "s510",
+    "s511",
+    "s512",
+    "s513",
+    "s514",
+    "s515",
+    "s516",
+    "s517",
+    "s518",
+    "s519",
+    "s520",
+    "s521",
+    "s522",
+    "s523",
+    "s524",
+    "s525",
+    "s526",
+    "s527",
+    "s528",
+    "s529",
+    "s530",
+    "s531",
+    "s532",
+    "s533",
+    "s534",
+    "s535",
+    "s536",
+    "s537",
+    "s538",
+    "s539",
+    "s540",
+    "s541",
+    "s542",
+    "s543",
+    "s544",
+    "s545",
+    "s546",
+    "s547",
+    "s548",
+    "s549",
+    "s550",
+    "s551",
+    "s552",
+    "s553",
+    "s554",
+    "s555",
+    "s556",
+    "s557",
+    "s558",
+    "s559",
+    "s560",
+    "s561",
+    "s562",
+    "s563",
+    "s564",
+    "s565",
+    "s566",
+    "s567",
+    "s568",
+    "s569",
+    "s570",
+    "s571",
+    "s572",
+    "s573",
+    "s574",
+    "s575",
+    "s576",
+    "s577",
+    "s578",
+    "s579",
+    "s580",
+    "s581",
+    "s582",
+    "s583",
+    "s584",
+    "s585",
+    "s586",
+    "s587",
+    "s588",
+    "s589",
+    "s590",
+    "s591",
+    "s592",
+    "s593",
+    "s594",
+    "s595",
+    "s596",
+    "s597",
+    "s598",
+    "s599",
+    "s600",
+    "s601",
+    "s602",
+    "s603",
+    "s604",
+    "s605",
+    "s606",
+    "s607",
+    "s608",
+    "s609",
+    "s610",
+    "s611",
+    "s612",
+    "s613",
+    "s614",
+    "s615",
+    "s616",
+    "s617",
+    "s618",
+    "s619",
+    "s620",
+    "s621",
+    "s622",
+    "s623",
+    "s624",
+    "s625",
+    "s626",
+    "s627",
+    "s628",
+    "s629",
+    "s630",
+    "s631",
+    "s632",
+    "s633",
+    "s634",
+    "s635",
+    "s636",
+    "s637",
+    "s638",
+    "s639",
+    "s640",
+    "s641",
+    "s642",
+    "s643",
+    "s644",
+    "s645",
+    "s646",
+    "s647",
+    "s648",
+    "s649",
+    "s650",
+    "s651",
+    "s652",
+    "s653",
+    "s654",
+    "s655",
+    "s656",
+    "s657",
+    "s658",
+    "s659",
+    "s660",
+    "s661",
+    "s662",
+    "s663",
+    "s664",
+    "s665",
+    "s666",
+    "s667",
+    "s668",
+    "s669",
+    "s670",
+    "s671",
+    "s672",
+    "s673",
+    "s674",
+    "s675",
+    "s676",
+    "s677",
+    "s678",
+    "s679",
+    "s680",
+    "s681",
+    "s682",
+    "s683",
+    "s684",
+    "s685",
+    "s686",
+    "s687",
+    "s688",
+    "s689",
+    "s690",
+    "s691",
+    "s692",
+    "s693",
+    "s694",
+    "s695",
+    "s696",
+    "s697",
+    "s698",
+    "s699",
+    "s700",
+    "s701",
+    "s702",
+    "s703",
+    "s704",
+    "s705",
+    "s706",
+    "s707",
+    "s708",
+    "s709",
+    "s710",
+    "s711",
+    "s712",
+    "s713",
+    "s714",
+    "s715",
+    "s716",
+    "s717",
+    "s718",
+    "s719",
+    "s720",
+    "s721",
+    "s722",
+    "s723",
+    "s724",
+    "s725",
+    "s726",
+    "s727",
+    "s728",
+    "s729",
+    "s730",
+    "s731",
+    "s732",
+    "s733",
+    "s734",
+    "s735",
+    "s736",
+    "s737",
+    "s738",
+    "s739",
+    "s740",
+    "s741",
+    "s742",
+    "s743",
+    "s744",
+    "s745",
+    "s746",
+    "s747",
+    "s748",
+    "s749",
+    "s750",
+    "s751",
+    "s752",
+    "s753",
+    "s754",
+    "s755",
+    "s756",
+    "s757",
+    "s758",
+    "s759",
+    "s760",
+    "s761",
+    "s762",
+    "s763",
+    "s764",
+    "s765",
+    "s766",
+    "s767",
+    "s768",
+    "s769",
+    "s770",
+    "s771",
+    "s772",
+    "s773",
+    "s774",
+    "s775",
+    "s776",
+    "s777",
+    "s778",
+    "s779",
+    "s780",
+    "s781",
+    "s782",
+    "s783",
+    "s784",
+    "s785",
+    "s786",
+    "s787",
+    "s788",
+    "s789",
+    "s790",
+    "s791",
+    "s792",
+    "s793",
+    "s794",
+    "s795",
+    "s796",
+    "s797",
+    "s798",
+    "s799",
+    "s800",
+    "s801",
+    "s802",
+    "s803",
+    "s804",
+    "s805",
+    "s806",
+    "s807",
+    "s808",
+    "s809",
+    "s810",
+    "s811",
+    "s812",
+    "s813",
+    "s814",
+    "s815",
+    "s816",
+    "s817",
+    "s818",
+    "s819",
+    "s820",
+    "s821",
+    "s822",
+    "s823",
+    "s824",
+    "s825",
+    "s826",
+    "s827",
+    "s828",
+    "s829",
+    "s830",
+    "s831",
+    "s832",
+    "s833",
+    "s834",
+    "s835",
+    "s836",
+    "s837",
+    "s838",
+    "s839",
+    "s840",
+    "s841",
+    "s842",
+    "s843",
+    "s844",
+    "s845",
+    "s846",
+    "s847",
+    "s848",
+    "s849",
+    "s850",
+    "s851",
+    "s852",
+    "s853",
+    "s854",
+    "s855",
+    "s856",
+    "s857",
+    "s858",
+    "s859",
+    "s860",
+    "s861",
+    "s862",
+    "s863",
+    "s864",
+    "s865",
+    "s866",
+    "s867",
+    "s868",
+    "s869",
+    "s870",
+    "s871",
+    "s872",
+    "s873",
+    "s874",
+    "s875",
+    "s876",
+    "s877",
+    "s878",
+    "s879",
+    "s880",
+    "s881",
+    "s882",
+    "s883",
+    "s884",
+    "s885",
+    "s886",
+    "s887",
+    "s888",
+    "s889",
+    "s890",
+    "s891",
+    "s892",
+    "s893",
+    "s894",
+    "s895",
+    "s896",
+    "s897",
+    "s898",
+    "s899",
+    "s900",
+    "s901",
+    "s902",
+    "s903",
+    "s904",
+    "s905",
+    "s906",
+    "s907",
+    "s908",
+    "s909",
+    "s910",
+    "s911",
+    "s912",
+    "s913",
+    "s914",
+    "s915",
+    "s916",
+    "s917",
+    "s918",
+    "s919",
+    "s920",
+    "s921",
+    "s922",
+    "s923",
+    "s924",
+    "s925",
+    "s926",
+    "s927",
+    "s928",
+    "s929",
+    "s930",
+    "s931",
+    "s932",
+    "s933",
+    "s934",
+    "s935",
+    "s936",
+    "s937",
+    "s938",
+    "s939",
+    "s940",
+    "s941",
+    "s942",
+    "s943",
+    "s944",
+    "s945",
+    "s946",
+    "s947",
+    "s948",
+    "s949",
+    "s950",
+    "s951",
+    "s952",
+    "s953",
+    "s954",
+    "s955",
+    "s956",
+    "s957",
+    "s958",
+    "s959",
+    "s960",
+    "s961",
+    "s962",
+    "s963",
+    "s964",
+    "s965",
+    "s966",
+    "s967",
+    "s968",
+    "s969",
+    "s970",
+    "s971",
+    "s972",
+    "s973",
+    "s974",
+    "s975",
+    "s976",
+    "s977",
+    "s978",
+    "s979",
+    "s980",
+    "s981",
+    "s982",
+    "s983",
+    "s984",
+    "s985",
+    "s986",
+    "s987",
+    "s988",
+    "s989",
+    "s990",
+    "s991",
+    "s992",
+    "s993",
+    "s994",
+    "s995",
+    "s996",
+    "s997",
+    "s998",
+    "s999",
+    "<eop>",
+    "<eog>",
+    "<|begin_of_sentence|>",
+    "<|end_of_sentence|>",
+    "<|User|>",
+    "<|Assistant|>",
+    "<think>",
+    "</think>",
+    "<search_result>",
+    "</search_result>",
+    "<search_query>",
+    "</search_query>",
+    "<code_query>",
+    "</code_query>",
+    "<code_result>",
+    "</code_result>",
+    "<infer>",
+    "</infer>",
+    "<inferresult>",
+    "</inferresult>",
+    "<tool_calls>",
+    "</tool_calls>",
+    "<tool_response>",
+    "</tool_response>",
+    "<final_answer>",
+    "</final_answer>"
+  ],
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<eod>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<eod>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c84861a800c30e71099d63dca0963edbacf554586527ac037155a0560e2fb04
+size 14976038

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:36f79e0c70f73cdd2a8dd0fbe7bfe290da158eea746778d289e4ad76c8b383d9
+size 2155861

utils.py

@@ -0,0 +1,144 @@
+from typing import (Callable, Dict, Iterable, List, Literal, Mapping, Optional, Protocol, Set, Tuple, Union, overload)
+import torch
+from torch.func import functional_call
+
+@overload
+def flatten_bn(x: torch.Tensor) -> torch.Tensor:
+    ...
+
+
+@overload
+def flatten_bn(x: List[torch.Tensor]) -> List[torch.Tensor]:
+    ...
+
+
+@overload
+def flatten_bn(
+    x: Union[List[torch.Tensor], torch.Tensor],
+    *,
+    concat: Literal[True],
+) -> torch.Tensor:
+    ...
+
+
+@overload
+def flatten_bn(
+    x: Union[List[torch.Tensor], torch.Tensor],
+    *,
+    concat: bool = False,
+) -> Union[List[torch.Tensor], torch.Tensor]:
+    ...
+
+
+def flatten_bn(
+    x: Union[List[torch.Tensor], torch.Tensor],
+    *,
+    concat: bool = False,
+) -> Union[List[torch.Tensor], torch.Tensor]:
+    """
+    Flatten the ``B`` and ``N`` dimensions of batched multimodal inputs.
+
+    The input tensor should have shape ``(B, N, ...)```.
+    """
+    if isinstance(x, torch.Tensor):
+        return x.flatten(0, 1)
+
+    if concat:
+        return torch.cat(x)
+
+    return [x_n for x_b in x for x_n in x_b]
+
+def _flatten_embeddings(embeddings: torch.Tensor) -> torch.Tensor:
+    """
+    Recursively flattens and concatenates NestedTensors on all but the last
+    dimension.
+    """
+
+    if isinstance(embeddings, torch.Tensor):
+        # Flatten all but the last dimension.
+        return embeddings.flatten(0, -2)
+
+    return torch.cat(tuple(_flatten_embeddings(t) for t in embeddings))
+
+def _embedding_count_expression(embeddings: torch.Tensor) -> str:
+    """
+    Constructs a debugging representation of the number of embeddings in the
+    Tensors.
+    """
+
+    if isinstance(embeddings, torch.Tensor):
+        return " x ".join([str(dim) for dim in embeddings.shape[:-1]])
+
+    return " + ".join(
+        _embedding_count_expression(inner) for inner in embeddings)
+
+def _merge_multimodal_embeddings(
+    inputs_embeds: torch.Tensor,
+    is_multimodal: torch.Tensor,
+    multimodal_embeddings: torch.Tensor,
+) -> torch.Tensor:
+    """
+    Merge ``multimodal_embeddings`` into ``inputs_embeds`` by overwriting the
+    positions in ``inputs_embeds`` corresponding to placeholder tokens in
+    ``input_ids``.
+
+    Note:
+        This updates ``inputs_embeds`` in place.
+    """
+    num_expected_tokens = is_multimodal.sum().item()
+    assert isinstance(num_expected_tokens, int)
+    # [total_patches, text_config.hidden_size]
+    flattened = _flatten_embeddings(multimodal_embeddings)
+    if flattened.shape[0] != num_expected_tokens:
+        expr = _embedding_count_expression(multimodal_embeddings)
+        raise ValueError(
+            f"Attempted to assign {expr} = {flattened.shape[0]} "
+            f"multimodal tokens to {num_expected_tokens} placeholders")
+
+    inputs_embeds[is_multimodal] = flattened
+    return inputs_embeds
+
+def merge_multimodal_embeddings(
+    input_ids: torch.Tensor,
+    inputs_embeds: torch.Tensor,
+    multimodal_embeddings: torch.Tensor,
+    placeholder_token_id: Union[int, List[int]],
+) -> torch.Tensor:
+    """
+    Merge ``multimodal_embeddings`` into ``inputs_embeds`` by overwriting the
+    positions in ``inputs_embeds`` corresponding to placeholder tokens in
+    ``input_ids``.
+
+    ``placeholder_token_id`` can be a list of token ids (e.g, token ids
+    of img_start, img_break, and img_end tokens) when needed: This means
+    the order of these tokens in the ``input_ids`` MUST MATCH the order of
+    their embeddings in ``multimodal_embeddings`` since we need to
+    slice-merge instead of individually scattering.
+
+    For example, if input_ids is "TTTTTSIIIBIIIBIIIETTT", where
+    - T is text token
+    - S is image start token
+    - I is image embedding token
+    - B is image break token
+    - E is image end token.
+
+    Then the image embeddings (that correspond to I's) from vision encoder
+    must be padded with embeddings of S, B, and E in the same order of
+    input_ids for a correct embedding merge.
+
+    Note:
+        This updates ``inputs_embeds`` in place.
+    """
+    if isinstance(placeholder_token_id, list):
+        placeholder_token_id = torch.tensor(placeholder_token_id,
+                                            device=input_ids.device)
+        return _merge_multimodal_embeddings(
+            inputs_embeds,
+            torch.isin(input_ids, placeholder_token_id),
+            multimodal_embeddings,
+        )
+    return _merge_multimodal_embeddings(
+        inputs_embeds,
+        (input_ids == placeholder_token_id),
+        multimodal_embeddings,
+    )