anlp-assignment1-checkpoints / tokeniser /source_bpe_scratch_1024.json
Upload tokeniser/source_bpe_scratch_1024.json with huggingface_hub
11ee947 verified 2 months ago
19.3 kB
{"format":"anlp_bpe_from_scratch_v1","mode":"source","special_tokens":["[PAD]","[BOS]","[EOS]","[UNK]"],"id_to_token":["[PAD]","[BOS]","[EOS]","[UNK]","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","",""],"merges":[[45,116,260],[89,22,261],[40,114,262],[91,26,263],[91,20,264],[87,101,265],[47,112,266],[57,22,267],[22,72,268],[22,70,269],[69,26,270],[67,116,271],[65,112,272],[54,114,273],[69,20,274],[73,101,275],[40,94,276],[74,98,277],[74,92,278],[62,40,279],[60,60,280],[57,42,281],[46,112,282],[44,116,283],[90,26,284],[41,114,285],[90,20,286],[86,101,287],[39,22,288],[71,22,289],[36,112,290],[38,116,291],[96,26,292],[51,114,293],[87,70,294],[96,20,295],[91,72,296],[41,66,297],[92,101,298],[87,72,299],[91,70,300],[95,98,301],[88,22,302],[56,22,303],[268,94,304],[269,45,305],[47,66,306],[87,98,307],[49,66,308],[45,38,309],[57,68,310],[74,93,311],[89,68,312],[40,64,313],[87,55,314],[74,99,315],[62,112,316],[74,26,317],[57,114,318],[60,116,319],[74,20,320],[276,261,321],[277,262,322],[96,91,323],[278,263,324],[281,260,325],[97,98,326],[280,264,327],[51,38,328],[279,267,329],[49,40,330],[39,66,331],[43,38,332],[97,70,333],[304,265,334],[95,92,335],[44,36,336],[36,64,337],[305,266,338],[36,36,339],[85,96,340],[49,38,341],[87,94,342],[85,98,343],[87,92,344],[91,51,345],[53,96,346],[22,93,347],[99,51,348],[97,72,349],[51,60,350],[40,282,351],[68,93,352],[50,64,353],[57,116,354],[83,101,355],[45,66,356],[47,283,357],[57,286,358],[89,284,359],[59,112,360],[79,26,361],[87,285,362],[70,101,363],[39,38,364],[60,114,365],[99,68,366],[95,72,367],[79,20,368],[91,287,369],[22,99,370],[37,66,371],[68,99,372],[91,98,373],[96,85,374],[95,70,375],[67,68,376],[85,99,377],[36,38,378],[51,66,379],[22,91,380],[69,72,381],[68,87,382],[87,96,383],[91,302,384],[97,84,385],[68,89,386],[37,112,387],[39,116,388],[67,40,389],[55,47,390],[50,114,391],[97,73,392],[49,64,393],[22,85,394],[97,20,395],[97,71,396],[45,303,397],[53,98,398],[97,26,399],[53,68,400],[44,112,401],[37,61,402],[93,55,403],[91,55,404],[47,38,405],[97,90,406],[37,46,407],[43,114,408],[67,88,409],[97,92,410],[45,22,411],[39,58,412],[50,401,413],[88,26,414],[37,38,415],[39,41,416],[101,329,417],[50,36,418],[85,93,419],[60,67,420],[40,36,421],[57,96,422],[77,22,423],[51,67,424],[91,92,425],[407,116,426],[85,68,427],[87,51,428],[75,72,429],[95,89,430],[69,93,431],[57,43,432],[94,93,433],[409,20,434],[49,116,435],[345,282,436],[42,114,437],[87,26,438],[36,114,439],[36,62,440],[87,20,441],[89,20,442],[308,286,443],[61,96,444],[45,112,445],[339,283,446],[42,260,447],[49,39,448],[64,45,449],[51,36,450],[344,285,451],[51,112,452],[310,20,453],[307,284,454],[47,116,455],[37,291,456],[89,26,457],[89,96,458],[66,57,459],[313,112,460],[340,287,461],[312,26,462],[49,114,463],[306,116,464],[69,99,465],[332,59,466],[61,323,467],[98,93,468],[314,114,469],[97,298,470],[296,101,471],[38,91,472],[50,290,473],[67,295,474],[98,99,475],[95,87,476],[94,99,477],[50,63,478],[44,290,479],[49,43,480],[43,291,481],[45,114,482],[335,437,483],[297,442,484],[91,94,485],[22,73,486],[95,298,487],[69,70,488],[374,101,489],[61,295,490],[346,302,491],[25,447,492],[22,88,493],[95,99,494],[336,455,495],[41,40,496],[22,86,497],[328,303,498],[74,72,499],[75,70,500],[97,95,501],[301,457,502],[22,71,503],[57,387,504],[54,62,505],[85,72,506],[22,69,507],[40,61,508],[89,98,509],[62,388,510],[45,43,511],[65,60,512],[91,87,513],[74,70,514],[95,93,515],[40,395,516],[53,274,517],[36,272,518],[66,67,519],[87,71,520],[22,75,521],[91,91,522],[51,271,523],[85,70,524],[22,89,525],[348,445,526],[91,89,527],[22,98,528],[44,62,529],[39,64,530],[87,275,531],[47,271,532],[40,272,533],[61,326,534],[95,71,535],[91,73,536],[22,87,537],[311,101,538],[45,39,539],[51,37,540],[41,55,541],[57,274,542],[22,92,543],[385,101,544],[99,93,545],[85,92,546],[91,71,547],[87,95,548],[90,93,549],[44,38,550],[73,98,551],[55,37,552],[97,97,553],[87,289,554],[41,39,555],[56,64,556],[91,275,557],[269,391,558],[91,273,559],[85,270,560],[268,399,561],[49,288,562],[73,99,563],[43,60,564],[44,49,565],[36,39,566],[96,87,567],[99,99,568],[45,288,569],[41,64,570],[87,93,571],[67,96,572],[63,66,573],[94,26,574],[95,96,575],[91,57,576],[64,39,577],[40,38,578],[45,40,579],[38,42,580],[91,85,581],[91,101,582],[91,289,583],[370,414,584],[69,71,585],[95,73,586],[73,93,587],[347,408,588],[40,65,589],[99,20,590],[67,61,591],[89,270,592],[45,266,593],[95,85,594],[36,61,595],[69,73,596],[22,74,597],[48,114,598],[95,94,599],[22,68,600],[50,39,601],[47,60,602],[94,265,603],[45,64,604],[43,51,605],[87,73,606],[91,88,607],[87,97,608],[422,72,609],[45,53,610],[51,291,611],[90,99,612],[44,65,613],[75,98,614],[36,45,615],[342,101,616],[49,65,617],[97,74,618],[56,36,619],[66,61,620],[88,93,621],[22,84,622],[49,55,623],[99,293,624],[68,91,625],[90,87,626],[343,26,627],[36,290,628],[85,85,629],[50,38,630],[70,93,631],[89,99,632],[87,88,633],[378,112,634],[44,456,635],[57,64,636],[91,96,637],[69,87,638],[87,74,639],[75,92,640],[94,20,641],[87,86,642],[36,316,643],[276,20,644],[494,292,645],[405,40,646],[279,116,647],[432,331,648],[44,60,649],[37,116,650],[53,295,651],[450,116,652],[36,65,653],[95,68,654],[46,61,655],[93,408,656],[53,74,657],[45,56,658],[44,272,659],[22,436,660],[54,43,661],[95,74,662],[96,93,663],[96,99,664],[515,293,665],[96,70,666],[37,36,667],[69,363,668],[88,366,669],[22,104,670],[41,474,671],[87,298,672],[22,90,673],[51,62,674],[398,20,675],[40,267,676],[69,89,677],[376,20,678],[51,319,679],[22,461,680],[91,74,681],[57,98,682],[58,333,683],[44,61,684],[38,41,685],[53,320,686],[54,316,687],[353,112,688],[74,87,689],[281,112,690],[70,99,691],[371,116,692],[70,70,693],[73,559,694],[94,261,695],[91,95,696],[99,414,697],[22,102,698],[70,72,699],[50,61,700],[51,51,701],[40,49,702],[36,116,703],[95,275,704],[88,99,705],[88,403,706],[91,293,707],[60,264,708],[93,292,709],[97,75,710],[63,517,711],[22,117,712],[49,360,713],[22,95,714],[87,68,715],[41,38,716],[73,560,717],[97,96,718],[97,68,719],[51,64,720],[98,85,721],[277,101,722],[69,98,723],[87,89,724],[44,64,725],[44,364,726],[69,92,727],[91,68,728],[62,61,729],[39,39,730],[44,46,731],[96,89,732],[45,55,733],[22,123,734],[95,289,735],[51,47,736],[49,36,737],[55,43,738],[85,317,739],[22,105,740],[294,101,741],[106,92,742],[58,523,743],[91,318,744],[63,60,745],[95,95,746],[45,65,747],[51,61,748],[90,89,749],[42,41,750],[22,103,751],[97,94,752],[106,98,753],[267,72,754],[72,93,755],[74,91,756],[75,94,757],[88,72,758],[95,86,759],[98,26,760],[41,47,761],[97,69,762],[261,70,763],[22,97,764],[91,298,765],[44,57,766],[413,321,767],[75,71,768],[99,26,769],[37,67,770],[43,66,771],[89,92,772],[98,20,773],[99,85,774],[406,338,775],[22,118,776],[38,62,777],[43,67,778],[95,88,779],[434,322,780],[71,55,781],[57,295,782],[74,362,783],[426,324,784],[40,39,785],[87,273,786],[43,554,787],[370,293,788],[49,56,789],[347,292,790],[56,38,791],[61,562,792],[63,38,793],[85,91,794],[22,120,795],[41,68,796],[98,91,797],[44,39,798],[98,355,799],[73,68,800],[36,67,801],[47,67,802],[385,417,803],[349,101,804],[22,121,805],[53,311,806],[60,38,807],[380,101,808],[96,361,809],[412,334,810],[36,57,811],[65,41,812],[36,46,813],[40,290,814],[57,272,815],[73,92,816],[36,49,817],[39,43,818],[87,85,819],[74,275,820],[22,124,821],[75,73,822],[84,87,823],[49,266,824],[61,40,825],[93,20,826],[63,61,827],[95,97,828],[22,119,829],[47,291,830],[98,89,831],[63,67,832],[22,129,833],[51,40,834],[46,114,835],[49,37,836],[60,61,837],[88,70,838],[66,53,839],[50,57,840],[68,85,841],[58,42,842],[72,99,843],[84,89,844],[30,676,845],[94,365,846],[54,47,847],[64,41,848],[36,53,849],[40,60,850],[40,62,851],[48,66,852],[22,127,853],[42,300,854],[50,65,855],[57,357,856],[22,130,857],[38,37,858],[315,473,859],[53,38,860],[75,96,861],[97,93,862],[742,263,863],[56,65,864],[51,46,865],[61,41,866],[74,85,867],[94,91,868],[61,66,869],[346,88,870],[44,41,871],[49,47,872],[57,47,873],[74,359,874],[67,57,875],[43,280,876],[98,87,877],[55,114,878],[74,289,879],[98,94,880],[8,695,881],[96,86,882],[38,415,883],[42,42,884],[328,56,885],[39,47,886],[49,59,887],[61,274,888],[753,262,889],[43,367,890],[98,265,891],[38,266,892],[458,70,893],[22,125,894],[51,57,895],[54,42,896],[74,89,897],[309,264,898],[49,63,899],[89,93,900],[343,101,901],[64,38,902],[342,26,903],[50,66,904],[49,61,905],[428,62,906],[350,534,907],[98,261,908],[330,116,909],[54,65,910],[507,518,911],[580,112,912],[424,96,913],[12,290,914],[28,708,915],[50,41,916],[40,53,917],[51,44,918],[276,739,919],[22,94,920],[39,65,921],[827,278,922],[67,67,923],[91,97,924],[43,271,925],[40,300,926],[46,38,927],[85,89,928],[521,531,929],[96,73,930],[373,317,931],[95,91,932],[51,43,933],[41,43,934],[45,47,935],[84,93,936],[94,87,937],[22,126,938],[85,261,939],[55,326,940],[84,355,941],[36,411,942],[68,71,943],[296,262,944],[68,26,945],[45,61,946],[51,63,947],[54,266,948],[58,876,949],[67,60,950],[74,265,951],[91,93,952],[54,63,953],[74,469,954],[300,263,955],[314,266,956],[312,265,957],[669,26,958],[22,131,959],[41,65,960],[44,647,961],[48,266,962],[277,643,963],[313,260,964],[341,112,965],[280,299,966],[50,60,967],[890,574,968],[866,644,969],[421,319,970],[356,320,971],[376,72,972],[706,114,973],[46,41,974],[89,72,975],[89,85,976],[36,60,977],[43,692,978],[49,41,979],[51,445,980],[98,96,981],[36,41,982],[39,63,983],[74,314,984],[74,312,985],[297,334,986],[36,267,987],[39,360,988],[46,678,989],[87,87,990],[279,686,991],[48,354,992],[69,265,993],[85,95,994],[53,99,995],[61,38,996],[68,73,997],[281,330,998],[49,331,999],[51,337,1000],[51,297,1001],[85,265,1002],[97,88,1003],[279,57,1004],[40,66,1005],[91,99,1006],[487,329,1007],[70,94,1008],[87,287,1009],[91,75,1010],[97,86,1011],[479,321,1012],[48,39,1013],[97,101,1014],[481,324,1015],[22,128,1016],[59,266,1017],[375,482,1018],[57,306,1019],[73,26,1020],[304,744,1021],[36,66,1022],[61,39,1023]]}