tokenizer

removed.jsonl

@@ -0,0 +1,230 @@
+{"in": "<nl>", "out": "記録を樹立", "idx": 30635, "score": -12.677860260009766}
+{"in": "<tab>", "out": "▁978-4-253-", "idx": 30608, "score": -12.674758911132812}
+{"in": "<sep>", "out": "2019-20", "idx": 30599, "score": -12.673821449279785}
+{"in": "<code>", "out": "を認識し", "idx": 30592, "score": -12.67286491394043}
+{"in": "</code>", "out": "を説得し", "idx": 30543, "score": -12.666202545166016}
+{"in": "False", "out": "周年を迎えた", "idx": 30541, "score": -12.665994644165039}
+{"in": "None", "out": "さいとう", "idx": 30526, "score": -12.66370964050293}
+{"in": "True", "out": "を向上させ", "idx": 30522, "score": -12.663201332092285}
+{"in": "assert", "out": "ことが発覚", "idx": 30518, "score": -12.662910461425781}
+{"in": "async", "out": "にノミネートされた", "idx": 30516, "score": -12.66283893585205}
+{"in": "await", "out": "を操作する", "idx": 30508, "score": -12.661879539489746}
+{"in": "continue", "out": "涙を流し", "idx": 30505, "score": -12.661367416381836}
+{"in": "elif", "out": "以下の分離集落", "idx": 30503, "score": -12.661301612854004}
+{"in": "else", "out": "を本拠地とする", "idx": 30502, "score": -12.661028861999512}
+{"in": "except", "out": "一面もある", "idx": 30501, "score": -12.660845756530762}
+{"in": "finally", "out": "を経営する", "idx": 30489, "score": -12.6598539352417}
+{"in": "import", "out": "を批判する", "idx": 30486, "score": -12.659765243530273}
+{"in": "nonlocal", "out": "に派遣し", "idx": 30483, "score": -12.659488677978516}
+{"in": "raise", "out": "が進行し", "idx": 30476, "score": -12.65832805633545}
+{"in": "while", "out": "教授に就任", "idx": 30464, "score": -12.65697956085205}
+{"in": "yield", "out": "に違反し", "idx": 30450, "score": -12.655882835388184}
+{"in": "▁False", "out": "眼鏡をかけ", "idx": 30439, "score": -12.654056549072266}
+{"in": "▁None", "out": "を入力し", "idx": 30434, "score": -12.653138160705566}
+{"in": "▁True", "out": "を成し遂げた", "idx": 30431, "score": -12.652752876281738}
+{"in": "▁async", "out": "^{2}+", "idx": 30422, "score": -12.651905059814453}
+{"in": "▁await", "out": "人気を集め", "idx": 30421, "score": -12.651870727539062}
+{"in": "▁else", "out": "面倒を見", "idx": 30409, "score": -12.650174140930176}
+{"in": "▁if", "out": "が禁止され", "idx": 30389, "score": -12.648188591003418}
+{"in": "▁import", "out": "呼び方", "idx": 30388, "score": -12.648162841796875}
+{"in": "▁lambda", "out": "に閉じ込められ", "idx": 30378, "score": -12.647259712219238}
+{"in": "~", "out": "が新設され", "idx": 30368, "score": -12.64654541015625}
+{"in": "//", "out": "に在籍し", "idx": 30355, "score": -12.644821166992188}
+{"in": "<<", "out": "20,000", "idx": 30329, "score": -12.642166137695312}
+{"in": ">>", "out": "が挙げられ", "idx": 30325, "score": -12.641738891601562}
+{"in": "<=", "out": "2012-13", "idx": 30324, "score": -12.641613006591797}
+{"in": ">=", "out": "に注目し", "idx": 30319, "score": -12.640992164611816}
+{"in": "!=", "out": "を追求し", "idx": 30318, "score": -12.640813827514648}
+{"in": "==", "out": "ダメージを与え", "idx": 30316, "score": -12.640597343444824}
+{"in": ":=", "out": "の多さ", "idx": 30308, "score": -12.639910697937012}
+{"in": "▁+", "out": "を身に付け", "idx": 30287, "score": -12.637303352355957}
+{"in": "▁-", "out": "に吸収合併され", "idx": 30285, "score": -12.636656761169434}
+{"in": "▁~", "out": "を乗せて", "idx": 30249, "score": -12.632139205932617}
+{"in": "▁*", "out": "助けを求め", "idx": 30245, "score": -12.63174819946289}
+{"in": "▁**", "out": "に変換する", "idx": 30231, "score": -12.630305290222168}
+{"in": "▁@", "out": "を目的とし", "idx": 30221, "score": -12.629398345947266}
+{"in": "▁/", "out": "を禁止し", "idx": 30218, "score": -12.629301071166992}
+{"in": "▁//", "out": "ような気がします", "idx": 30216, "score": -12.629130363464355}
+{"in": "▁%", "out": "を排除し", "idx": 30212, "score": -12.628326416015625}
+{"in": "▁<<", "out": "とコメントした", "idx": 30210, "score": -12.628182411193848}
+{"in": "▁>>", "out": "ごく一部", "idx": 30207, "score": -12.627854347229004}
+{"in": "▁&", "out": "わかりやすい", "idx": 30190, "score": -12.626666069030762}
+{"in": "▁^", "out": "じゃないですか", "idx": 30159, "score": -12.623470306396484}
+{"in": "▁|", "out": "全く同じ", "idx": 30157, "score": -12.623373031616211}
+{"in": "▁<", "out": "の背後に", "idx": 30154, "score": -12.622865676879883}
+{"in": "▁<=", "out": "から指名され", "idx": 30151, "score": -12.622299194335938}
+{"in": "▁>=", "out": "があげられる", "idx": 30139, "score": -12.620767593383789}
+{"in": "▁!=", "out": "元に戻", "idx": 30135, "score": -12.620353698730469}
+{"in": "▁==", "out": "に接近し", "idx": 30133, "score": -12.619889259338379}
+{"in": "▁:=", "out": "者もいた", "idx": 30127, "score": -12.619523048400879}
+{"in": "S", "out": "���供給する", "idx": 30117, "score": -12.618501663208008}
+{"in": "P", "out": "最初期の", "idx": 30115, "score": -12.61833667755127}
+{"in": "instance", "out": "年代に入ると", "idx": 30111, "score": -12.618019104003906}
+{"in": "params", "out": "を模索し", "idx": 30102, "score": -12.617226600646973}
+{"in": "values", "out": "の学位を取得", "idx": 30100, "score": -12.617137908935547}
+{"in": "D", "out": "2018-19", "idx": 30091, "score": -12.616652488708496}
+{"in": "E", "out": "が建立され", "idx": 30087, "score": -12.616292953491211}
+{"in": "L", "out": "んかい", "idx": 30079, "score": -12.61522102355957}
+{"in": "A", "out": "学科を卒業", "idx": 30075, "score": -12.614690780639648}
+{"in": "B", "out": "を投入し", "idx": 30063, "score": -12.613547325134277}
+{"in": "items", "out": "を対象とし", "idx": 30055, "score": -12.61252212524414}
+{"in": "keys", "out": "いくつもの", "idx": 30053, "score": -12.612289428710938}
+{"in": "F", "out": "に乗せて", "idx": 30049, "score": -12.611647605895996}
+{"in": "G", "out": "を共有する", "idx": 30044, "score": -12.61114501953125}
+{"in": "names", "out": "を指定する", "idx": 30027, "score": -12.609272003173828}
+{"in": "status", "out": "が終わると", "idx": 30023, "score": -12.609105110168457}
+{"in": "axis", "out": "名前の由来", "idx": 30019, "score": -12.6089448928833}
+{"in": "R", "out": "それ以前に", "idx": 30006, "score": -12.607861518859863}
+{"in": "options", "out": "を確立し", "idx": 30003, "score": -12.607490539550781}
+{"in": "files", "out": "容疑で逮捕", "idx": 29999, "score": -12.607202529907227}
+{"in": "I", "out": "詳細は不明", "idx": 29994, "score": -12.607051849365234}
+{"in": "V", "out": "作目のシングル", "idx": 29969, "score": -12.605241775512695}
+{"in": "headers", "out": "しばらくの間", "idx": 29965, "score": -12.604829788208008}
+{"in": "O", "out": "を管轄する", "idx": 29953, "score": -12.603326797485352}
+{"in": "lines", "out": "よろしくお願いします", "idx": 29944, "score": -12.602762222290039}
+{"in": "types", "out": "現在地に移転", "idx": 29927, "score": -12.601107597351074}
+{"in": "copy", "out": "▁現行行政地名は", "idx": 29899, "score": -12.598560333251953}
+{"in": "H", "out": "国の史跡", "idx": 29898, "score": -12.598423957824707}
+{"in": "W", "out": "を流用し", "idx": 29895, "score": -12.597951889038086}
+{"in": "numpy", "out": "多く見られる", "idx": 29889, "score": -12.5968599319458}
+{"in": "fields", "out": "を発生させ", "idx": 29877, "score": -12.595332145690918}
+{"in": "results", "out": "出身のプロサッカー選手", "idx": 29867, "score": -12.59421157836914}
+{"in": "ids", "out": "が収録され", "idx": 29854, "score": -12.593358993530273}
+{"in": "columns", "out": "衝撃を受け", "idx": 29852, "score": -12.593182563781738}
+{"in": "logging", "out": "2008-09", "idx": 29848, "score": -12.592862129211426}
+{"in": "opts", "out": "▁978-4-3", "idx": 29838, "score": -12.591743469238281}
+{"in": "utils", "out": "注目を浴び", "idx": 29826, "score": -12.590426445007324}
+{"in": "Y", "out": "を保持する", "idx": 29824, "score": -12.590254783630371}
+{"in": "dict", "out": "}^{\\", "idx": 29812, "score": -12.589284896850586}
+{"in": "settings", "out": "続きを読む", "idx": 29811, "score": -12.5891695022583}
+{"in": "U", "out": "を占領し", "idx": 29809, "score": -12.58900260925293}
+{"in": "labels", "out": "行動を共にする", "idx": 29808, "score": -12.588854789733887}
+{"in": "sorted", "out": "ことが出来ない", "idx": 29806, "score": -12.588719367980957}
+{"in": "bytes", "out": "を研究する", "idx": 29789, "score": -12.58699893951416}
+{"in": "enumerate", "out": "取扱を開始", "idx": 29785, "score": -12.586787223815918}
+{"in": "K", "out": "こともしばしば", "idx": 29782, "score": -12.586579322814941}
+{"in": "nodes", "out": "効率的に", "idx": 29776, "score": -12.586114883422852}
+{"in": "The", "out": "博士号を取得", "idx": 29771, "score": -12.58564567565918}
+{"in": "ET", "out": "2014-15", "idx": 29766, "score": -12.584396362304688}
+{"in": "Q", "out": "})}", "idx": 29751, "score": -12.58310604095459}
+{"in": "X", "out": "▁978-4-4", "idx": 29745, "score": -12.582242012023926}
+{"in": "rows", "out": "権を獲得", "idx": 29736, "score": -12.581231117248535}
+{"in": "RE", "out": "気を付け", "idx": 29728, "score": -12.580394744873047}
+{"in": "objects", "out": "に同行し", "idx": 29716, "score": -12.579130172729492}
+{"in": "ext", "out": "に反発し", "idx": 29701, "score": -12.577690124511719}
+{"in": "points", "out": "攻撃を仕掛け", "idx": 29690, "score": -12.576614379882812}
+{"in": "parameters", "out": "23:30", "idx": 29689, "score": -12.576469421386719}
+{"in": "exists", "out": "に到達する", "idx": 29674, "score": -12.574638366699219}
+{"in": "startswith", "out": "2013-14", "idx": 29669, "score": -12.573983192443848}
+{"in": "models", "out": "を向上させる", "idx": 29670, "score": -12.573983192443848}
+{"in": "errors", "out": "空飛ぶ", "idx": 29658, "score": -12.573144912719727}
+{"in": "IN", "out": "駅に停車", "idx": 29654, "score": -12.572796821594238}
+{"in": "stats", "out": "ことが原因", "idx": 29653, "score": -12.572787284851074}
+{"in": "plot", "out": "事故を起こし", "idx": 29652, "score": -12.572601318359375}
+{"in": "Re", "out": "書を提出", "idx": 29649, "score": -12.572381973266602}
+{"in": "cols", "out": "の完全子会社", "idx": 29643, "score": -12.571518898010254}
+{"in": "Error", "out": "駅として開業", "idx": 29641, "score": -12.57137393951416}
+{"in": "tags", "out": "とも言われ", "idx": 29629, "score": -12.570301055908203}
+{"in": "indices", "out": "を発症し", "idx": 29628, "score": -12.570280075073242}
+{"in": "zeros", "out": "を制御する", "idx": 29619, "score": -12.569225311279297}
+{"in": "Not", "out": "を抑制する", "idx": 29615, "score": -12.568807601928711}
+{"in": "paths", "out": "を乗せた", "idx": 29603, "score": -12.567150115966797}
+{"in": "IO", "out": "とは思えない", "idx": 29567, "score": -12.563475608825684}
+{"in": "To", "out": "姿を消し", "idx": 29563, "score": -12.563262939453125}
+{"in": "verbose", "out": "ことを明らかにした", "idx": 29558, "score": -12.562886238098145}
+{"in": "groups", "out": "その名の通り", "idx": 29551, "score": -12.562413215637207}
+{"in": "attrs", "out": "^^;", "idx": 29547, "score": -12.562051773071289}
+{"in": "size", "out": "にある公立中学校", "idx": 29532, "score": -12.560529708862305}
+{"in": "vals", "out": "とコメントしている", "idx": 29511, "score": -12.557933807373047}
+{"in": "directory", "out": "わかりやすく", "idx": 29506, "score": -12.557350158691406}
+{"in": "parts", "out": "ことはできず", "idx": 29501, "score": -12.557062149047852}
+{"in": "subprocess", "out": "で始まるページの一覧タイトル", "idx": 29485, "score": -12.555493354797363}
+{"in": "ER", "out": "9,000", "idx": 29472, "score": -12.554306983947754}
+{"in": "features", "out": "から派生した", "idx": 29463, "score": -12.552811622619629}
+{"in": "requests", "out": "問題視され", "idx": 29452, "score": -12.551846504211426}
+{"in": "tokens", "out": "えもん", "idx": 29444, "score": -12.550905227661133}
+{"in": "axes", "out": "と推測されている", "idx": 29443, "score": -12.550715446472168}
+{"in": "flags", "out": "の項を参照", "idx": 29431, "score": -12.549659729003906}
+{"in": "samples", "out": "に分布し", "idx": 29421, "score": -12.548849105834961}
+{"in": "apply", "out": "を調査し", "idx": 29398, "score": -12.546494483947754}
+{"in": "ize", "out": "ことが示唆され", "idx": 29393, "score": -12.546011924743652}
+{"in": "IT", "out": "にすぎない", "idx": 29387, "score": -12.5451078414917}
+{"in": "weights", "out": "が適用され", "idx": 29372, "score": -12.543537139892578}
+{"in": "AL", "out": "から解放され", "idx": 29369, "score": -12.543176651000977}
+{"in": "AR", "out": "が死去し", "idx": 29358, "score": -12.54233455657959}
+{"in": "ED", "out": "防止のため", "idx": 29351, "score": -12.540937423706055}
+{"in": "propertie", "out": "なおかつ", "idx": 29347, "score": -12.540244102478027}
+{"in": "ctypes", "out": "たいと思う", "idx": 29331, "score": -12.538861274719238}
+{"in": "ST", "out": "を把握し", "idx": 29328, "score": -12.538528442382812}
+{"in": "AN", "out": "を運営している", "idx": 29319, "score": -12.537568092346191}
+{"in": "attribute", "out": "計画を立て", "idx": 29318, "score": -12.537273406982422}
+{"in": "split", "out": "が発達し", "idx": 29317, "score": -12.537243843078613}
+{"in": "vars", "out": "若い女性", "idx": 29298, "score": -12.535577774047852}
+{"in": "TYPE", "out": "と話した", "idx": 29297, "score": -12.535531997680664}
+{"in": "words", "out": "ことが分かる", "idx": 29288, "score": -12.535032272338867}
+{"in": "Class", "out": "世界で最も", "idx": 29281, "score": -12.534507751464844}
+{"in": "variable", "out": "々しく", "idx": 29279, "score": -12.534379959106445}
+{"in": "coords", "out": "を可能にし", "idx": 29273, "score": -12.534027099609375}
+{"in": "AT", "out": "を操作し", "idx": 29271, "score": -12.533834457397461}
+{"in": "seconds", "out": "ゆうちょ銀行", "idx": 29253, "score": -12.531882286071777}
+{"in": "ify", "out": "ことも多く", "idx": 29234, "score": -12.529809951782227}
+{"in": "parsed", "out": "ことを恐れ", "idx": 29219, "score": -12.528444290161133}
+{"in": "LOG", "out": "を宣言し", "idx": 29213, "score": -12.527778625488281}
+{"in": "Con", "out": "に配慮し", "idx": 29204, "score": -12.526726722717285}
+{"in": "counts", "out": "免許を取得", "idx": 29203, "score": -12.526590347290039}
+{"in": "comp", "out": "年契約で", "idx": 29184, "score": -12.524176597595215}
+{"in": "In", "out": "の悪さ", "idx": 29178, "score": -12.523634910583496}
+{"in": "dumps", "out": "を優先し", "idx": 29166, "score": -12.521889686584473}
+{"in": "obj", "out": "を発射する", "idx": 29161, "score": -12.521334648132324}
+{"in": "inputs", "out": "多くの人々", "idx": 29153, "score": -12.520657539367676}
+{"in": "edges", "out": "月に開催された", "idx": 29147, "score": -12.520158767700195}
+{"in": "policy", "out": "に焦点を当て", "idx": 29143, "score": -12.519235610961914}
+{"in": "units", "out": "2,8", "idx": 29137, "score": -12.519079208374023}
+{"in": "methods", "out": "が予想され", "idx": 29136, "score": -12.519023895263672}
+{"in": "matches", "out": "に入れられ", "idx": 29131, "score": -12.518667221069336}
+{"in": "colors", "out": "を入手し", "idx": 29128, "score": -12.518204689025879}
+{"in": "required", "out": "寒さ", "idx": 29127, "score": -12.518065452575684}
+{"in": "iter", "out": "今のところ", "idx": 29118, "score": -12.517746925354004}
+{"in": "Qt", "out": "ケースもある", "idx": 29101, "score": -12.516353607177734}
+{"in": "property", "out": "ことを認め", "idx": 29092, "score": -12.515559196472168}
+{"in": "Return", "out": "がなされる", "idx": 29086, "score": -12.515178680419922}
+{"in": "loads", "out": "が強化され", "idx": 29079, "score": -12.513934135437012}
+{"in": "alias", "out": "に興味を持ち", "idx": 29069, "score": -12.512958526611328}
+{"in": "fetch", "out": "よく似た", "idx": 29052, "score": -12.51140308380127}
+{"in": "astype", "out": "を構築する", "idx": 29050, "score": -12.51114273071289}
+{"in": "hash", "out": "を前提に", "idx": 29045, "score": -12.510549545288086}
+{"in": "layers", "out": "(^^)", "idx": 29034, "score": -12.509492874145508}
+{"in": "dims", "out": "に感染し", "idx": 29031, "score": -12.509356498718262}
+{"in": "events", "out": "を希望する", "idx": 29029, "score": -12.50919246673584}
+{"in": "contents", "out": "んぐ", "idx": 29026, "score": -12.508731842041016}
+{"in": "actions", "out": "▁4-09-", "idx": 29010, "score": -12.507172584533691}
+{"in": "argument", "out": "活動を再開", "idx": 29001, "score": -12.506525993347168}
+{"in": "lat", "out": "することを決定", "idx": 28986, "score": -12.50487995147705}
+{"in": "elements", "out": "を武器に", "idx": 28979, "score": -12.50408935546875}
+{"in": "ay", "out": "の前に現れ", "idx": 28963, "score": -12.502606391906738}
+{"in": "endswith", "out": "ご自身", "idx": 28958, "score": -12.502312660217285}
+{"in": "filters", "out": "**", "idx": 28950, "score": -12.501532554626465}
+{"in": "ES", "out": "最も有名な", "idx": 28947, "score": -12.501126289367676}
+{"in": "NAME", "out": "▁97840", "idx": 28935, "score": -12.500229835510254}
+{"in": "safe", "out": "を確保するため", "idx": 28934, "score": -12.499862670898438}
+{"in": "exceptions", "out": "を発生させる", "idx": 28920, "score": -12.498503684997559}
+{"in": "format", "out": "恐れがある", "idx": 28913, "score": -12.4978666305542}
+{"in": "summary", "out": "}}{", "idx": 28911, "score": -12.497712135314941}
+{"in": "defaults", "out": "に対抗するため", "idx": 28902, "score": -12.49681568145752}
+{"in": "which", "out": "ご了承ください", "idx": 28865, "score": -12.49402141571045}
+{"in": "OR", "out": "を拠点とする", "idx": 28851, "score": -12.492277145385742}
+{"in": "DEFAULT", "out": "継続的に", "idx": 28843, "score": -12.491694450378418}
+{"in": "warnings", "out": "決勝に進出", "idx": 28841, "score": -12.49164867401123}
+{"in": "states", "out": "を決意する", "idx": 28823, "score": -12.49034309387207}
+{"in": "modules", "out": "事ができ", "idx": 28815, "score": -12.489692687988281}
+{"in": "loss", "out": "が経営する", "idx": 28810, "score": -12.48923397064209}
+{"in": "itertool", "out": "に吸収され", "idx": 28788, "score": -12.486899375915527}
+{"in": "EN", "out": "2,6", "idx": 28786, "score": -12.486804008483887}
+{"in": "My", "out": "の支援を受け", "idx": 28769, "score": -12.485313415527344}
+{"in": "arg", "out": "2,0", "idx": 28735, "score": -12.482708930969238}
+{"in": "Thi", "out": "を与えること", "idx": 28725, "score": -12.482075691223145}
+{"in": "check", "out": "差をつけ", "idx": 28705, "score": -12.480887413024902}
+{"in": "classes", "out": "枠で放送", "idx": 28690, "score": -12.479047775268555}
+{"in": "CLI", "out": "事故が発生", "idx": 28674, "score": -12.477832794189453}
+{"in": "LE", "out": "被害を受け", "idx": 28672, "score": -12.47778606414795}
+{"in": "verify", "out": "見たこと", "idx": 28668, "score": -12.47750186920166}

vocamaru_log.txt

@@ -0,0 +1,263 @@
+[新しいモデルの保存先] sno_py500
+[LOG] 特殊語彙 id=0 type=3 <pad> 0.0
+[LOG] 特殊語彙 id=1 type=3 </s> 0.0
+[LOG] 特殊語彙 id=2 type=2 <unk> 0.0
+[全語彙数] 31741
+[スコア統計]                   s
+count  31741.000000
+mean     -11.467278
+std        1.075664
+min      -20.360178
+25%      -12.145124
+50%      -11.626710
+75%      -10.941068
+max       -3.172917
+[LOG] 登録済み else
+[LOG] 登録済み False
+[置き換える語彙数] 230
+[重複語数] 5209 トリム数 0
+全角ゴミ 125
+半角ゴミ 508
+記号ゴミ 2
+[記号ゴミ] 635
+数字ゴミ 465
+[数字重複] 465
+[消去可能な字句] 6309
+[実際に置き換える語] 230
+{"in": "<nl>", "out": "記録を樹立", "idx": 30635, "score": -12.677860260009766}
+{"in": "<tab>", "out": "▁978-4-253-", "idx": 30608, "score": -12.674758911132812}
+{"in": "<sep>", "out": "2019-20", "idx": 30599, "score": -12.673821449279785}
+{"in": "<code>", "out": "を認識し", "idx": 30592, "score": -12.67286491394043}
+{"in": "</code>", "out": "を説得し", "idx": 30543, "score": -12.666202545166016}
+{"in": "False", "out": "周年を迎えた", "idx": 30541, "score": -12.665994644165039}
+{"in": "None", "out": "さいとう", "idx": 30526, "score": -12.66370964050293}
+{"in": "True", "out": "を向上させ", "idx": 30522, "score": -12.663201332092285}
+{"in": "assert", "out": "ことが発覚", "idx": 30518, "score": -12.662910461425781}
+{"in": "async", "out": "にノミネートされた", "idx": 30516, "score": -12.66283893585205}
+{"in": "await", "out": "を操作する", "idx": 30508, "score": -12.661879539489746}
+{"in": "continue", "out": "涙を流し", "idx": 30505, "score": -12.661367416381836}
+{"in": "elif", "out": "以下の分離集落", "idx": 30503, "score": -12.661301612854004}
+{"in": "else", "out": "を本拠地とする", "idx": 30502, "score": -12.661028861999512}
+{"in": "except", "out": "一面もある", "idx": 30501, "score": -12.660845756530762}
+{"in": "finally", "out": "を経営する", "idx": 30489, "score": -12.6598539352417}
+{"in": "import", "out": "を批判する", "idx": 30486, "score": -12.659765243530273}
+{"in": "nonlocal", "out": "に派遣し", "idx": 30483, "score": -12.659488677978516}
+{"in": "raise", "out": "が進行し", "idx": 30476, "score": -12.65832805633545}
+{"in": "while", "out": "教授に就任", "idx": 30464, "score": -12.65697956085205}
+{"in": "yield", "out": "に違反し", "idx": 30450, "score": -12.655882835388184}
+{"in": "▁False", "out": "眼鏡をかけ", "idx": 30439, "score": -12.654056549072266}
+{"in": "▁None", "out": "を入力し", "idx": 30434, "score": -12.653138160705566}
+{"in": "▁True", "out": "を成し遂げた", "idx": 30431, "score": -12.652752876281738}
+{"in": "▁async", "out": "^{2}+", "idx": 30422, "score": -12.651905059814453}
+{"in": "▁await", "out": "人気を集め", "idx": 30421, "score": -12.651870727539062}
+{"in": "▁else", "out": "面倒を見", "idx": 30409, "score": -12.650174140930176}
+{"in": "▁if", "out": "が禁止され", "idx": 30389, "score": -12.648188591003418}
+{"in": "▁import", "out": "呼び方", "idx": 30388, "score": -12.648162841796875}
+{"in": "▁lambda", "out": "に閉じ込められ", "idx": 30378, "score": -12.647259712219238}
+{"in": "~", "out": "が新設され", "idx": 30368, "score": -12.64654541015625}
+{"in": "//", "out": "に在籍し", "idx": 30355, "score": -12.644821166992188}
+{"in": "<<", "out": "20,000", "idx": 30329, "score": -12.642166137695312}
+{"in": ">>", "out": "が挙げられ", "idx": 30325, "score": -12.641738891601562}
+{"in": "<=", "out": "2012-13", "idx": 30324, "score": -12.641613006591797}
+{"in": ">=", "out": "に注目し", "idx": 30319, "score": -12.640992164611816}
+{"in": "!=", "out": "を追求し", "idx": 30318, "score": -12.640813827514648}
+{"in": "==", "out": "ダメージを与え", "idx": 30316, "score": -12.640597343444824}
+{"in": ":=", "out": "の多さ", "idx": 30308, "score": -12.639910697937012}
+{"in": "▁+", "out": "を身に付け", "idx": 30287, "score": -12.637303352355957}
+{"in": "▁-", "out": "に吸収合併され", "idx": 30285, "score": -12.636656761169434}
+{"in": "▁~", "out": "を乗せて", "idx": 30249, "score": -12.632139205932617}
+{"in": "▁*", "out": "助けを求め", "idx": 30245, "score": -12.63174819946289}
+{"in": "▁**", "out": "に変換する", "idx": 30231, "score": -12.630305290222168}
+{"in": "▁@", "out": "を目的とし", "idx": 30221, "score": -12.629398345947266}
+{"in": "▁/", "out": "を禁止し", "idx": 30218, "score": -12.629301071166992}
+{"in": "▁//", "out": "ような気がします", "idx": 30216, "score": -12.629130363464355}
+{"in": "▁%", "out": "を排除し", "idx": 30212, "score": -12.628326416015625}
+{"in": "▁<<", "out": "とコメントした", "idx": 30210, "score": -12.628182411193848}
+{"in": "▁>>", "out": "ごく一部", "idx": 30207, "score": -12.627854347229004}
+{"in": "▁&", "out": "わかりやすい", "idx": 30190, "score": -12.626666069030762}
+{"in": "▁^", "out": "じゃないですか", "idx": 30159, "score": -12.623470306396484}
+{"in": "▁|", "out": "全く同じ", "idx": 30157, "score": -12.623373031616211}
+{"in": "▁<", "out": "の背後に", "idx": 30154, "score": -12.622865676879883}
+{"in": "▁<=", "out": "から指名され", "idx": 30151, "score": -12.622299194335938}
+{"in": "▁>=", "out": "があげられる", "idx": 30139, "score": -12.620767593383789}
+{"in": "▁!=", "out": "元に戻", "idx": 30135, "score": -12.620353698730469}
+{"in": "▁==", "out": "に接近し", "idx": 30133, "score": -12.619889259338379}
+{"in": "▁:=", "out": "者もいた", "idx": 30127, "score": -12.619523048400879}
+{"in": "S", "out": "を供給する", "idx": 30117, "score": -12.618501663208008}
+{"in": "P", "out": "最初期の", "idx": 30115, "score": -12.61833667755127}
+{"in": "instance", "out": "年代に入ると", "idx": 30111, "score": -12.618019104003906}
+{"in": "params", "out": "を模索し", "idx": 30102, "score": -12.617226600646973}
+{"in": "values", "out": "の学位を取得", "idx": 30100, "score": -12.617137908935547}
+{"in": "D", "out": "2018-19", "idx": 30091, "score": -12.616652488708496}
+{"in": "E", "out": "が建立され", "idx": 30087, "score": -12.616292953491211}
+{"in": "L", "out": "んかい", "idx": 30079, "score": -12.61522102355957}
+{"in": "A", "out": "学科を卒業", "idx": 30075, "score": -12.614690780639648}
+{"in": "B", "out": "を投入し", "idx": 30063, "score": -12.613547325134277}
+{"in": "items", "out": "を対象とし", "idx": 30055, "score": -12.61252212524414}
+{"in": "keys", "out": "いくつもの", "idx": 30053, "score": -12.612289428710938}
+{"in": "F", "out": "に乗せて", "idx": 30049, "score": -12.611647605895996}
+{"in": "G", "out": "を共有する", "idx": 30044, "score": -12.61114501953125}
+{"in": "names", "out": "を指定する", "idx": 30027, "score": -12.609272003173828}
+{"in": "status", "out": "が終わると", "idx": 30023, "score": -12.609105110168457}
+{"in": "axis", "out": "名前の由来", "idx": 30019, "score": -12.6089448928833}
+{"in": "R", "out": "それ以前に", "idx": 30006, "score": -12.607861518859863}
+{"in": "options", "out": "を確立し", "idx": 30003, "score": -12.607490539550781}
+{"in": "files", "out": "容疑で逮捕", "idx": 29999, "score": -12.607202529907227}
+{"in": "I", "out": "詳細は不明", "idx": 29994, "score": -12.607051849365234}
+{"in": "V", "out": "作目のシングル", "idx": 29969, "score": -12.605241775512695}
+{"in": "headers", "out": "しばらくの間", "idx": 29965, "score": -12.604829788208008}
+{"in": "O", "out": "を管轄する", "idx": 29953, "score": -12.603326797485352}
+{"in": "lines", "out": "よろしくお願いします", "idx": 29944, "score": -12.602762222290039}
+{"in": "types", "out": "現在地に移転", "idx": 29927, "score": -12.601107597351074}
+{"in": "copy", "out": "▁現行行政地名は", "idx": 29899, "score": -12.598560333251953}
+{"in": "H", "out": "国の史跡", "idx": 29898, "score": -12.598423957824707}
+{"in": "W", "out": "を流用し", "idx": 29895, "score": -12.597951889038086}
+{"in": "numpy", "out": "多く見られる", "idx": 29889, "score": -12.5968599319458}
+{"in": "fields", "out": "を発生させ", "idx": 29877, "score": -12.595332145690918}
+{"in": "results", "out": "出身のプロサッカー選手", "idx": 29867, "score": -12.59421157836914}
+{"in": "ids", "out": "が収録され", "idx": 29854, "score": -12.593358993530273}
+{"in": "columns", "out": "衝撃を受け", "idx": 29852, "score": -12.593182563781738}
+{"in": "logging", "out": "2008-09", "idx": 29848, "score": -12.592862129211426}
+{"in": "opts", "out": "▁978-4-3", "idx": 29838, "score": -12.591743469238281}
+{"in": "utils", "out": "注目を浴び", "idx": 29826, "score": -12.590426445007324}
+{"in": "Y", "out": "を保持する", "idx": 29824, "score": -12.590254783630371}
+{"in": "dict", "out": "}^{\\", "idx": 29812, "score": -12.589284896850586}
+{"in": "settings", "out": "続きを読む", "idx": 29811, "score": -12.5891695022583}
+{"in": "U", "out": "を占領し", "idx": 29809, "score": -12.58900260925293}
+{"in": "labels", "out": "行動を共にする", "idx": 29808, "score": -12.588854789733887}
+{"in": "sorted", "out": "ことが出来ない", "idx": 29806, "score": -12.588719367980957}
+{"in": "bytes", "out": "を研究する", "idx": 29789, "score": -12.58699893951416}
+{"in": "enumerate", "out": "取扱を開始", "idx": 29785, "score": -12.586787223815918}
+{"in": "K", "out": "こともしばしば", "idx": 29782, "score": -12.586579322814941}
+{"in": "nodes", "out": "効率的に", "idx": 29776, "score": -12.586114883422852}
+{"in": "The", "out": "博士号を取得", "idx": 29771, "score": -12.58564567565918}
+{"in": "ET", "out": "2014-15", "idx": 29766, "score": -12.584396362304688}
+{"in": "Q", "out": "})}", "idx": 29751, "score": -12.58310604095459}
+{"in": "X", "out": "▁978-4-4", "idx": 29745, "score": -12.582242012023926}
+{"in": "rows", "out": "権を獲得", "idx": 29736, "score": -12.581231117248535}
+{"in": "RE", "out": "気を付け", "idx": 29728, "score": -12.580394744873047}
+{"in": "objects", "out": "に同行し", "idx": 29716, "score": -12.579130172729492}
+{"in": "ext", "out": "に反発し", "idx": 29701, "score": -12.577690124511719}
+{"in": "points", "out": "攻撃を仕掛け", "idx": 29690, "score": -12.576614379882812}
+{"in": "parameters", "out": "23:30", "idx": 29689, "score": -12.576469421386719}
+{"in": "exists", "out": "に到達する", "idx": 29674, "score": -12.574638366699219}
+{"in": "startswith", "out": "2013-14", "idx": 29669, "score": -12.573983192443848}
+{"in": "models", "out": "を向上させる", "idx": 29670, "score": -12.573983192443848}
+{"in": "errors", "out": "空飛ぶ", "idx": 29658, "score": -12.573144912719727}
+{"in": "IN", "out": "駅に停車", "idx": 29654, "score": -12.572796821594238}
+{"in": "stats", "out": "ことが原因", "idx": 29653, "score": -12.572787284851074}
+{"in": "plot", "out": "事故を起こし", "idx": 29652, "score": -12.572601318359375}
+{"in": "Re", "out": "書を提出", "idx": 29649, "score": -12.572381973266602}
+{"in": "cols", "out": "の完全子会社", "idx": 29643, "score": -12.571518898010254}
+{"in": "Error", "out": "駅として開業", "idx": 29641, "score": -12.57137393951416}
+{"in": "tags", "out": "とも言われ", "idx": 29629, "score": -12.570301055908203}
+{"in": "indices", "out": "を発症し", "idx": 29628, "score": -12.570280075073242}
+{"in": "zeros", "out": "を制御する", "idx": 29619, "score": -12.569225311279297}
+{"in": "Not", "out": "を抑制する", "idx": 29615, "score": -12.568807601928711}
+{"in": "paths", "out": "を乗せた", "idx": 29603, "score": -12.567150115966797}
+{"in": "IO", "out": "とは思えない", "idx": 29567, "score": -12.563475608825684}
+{"in": "To", "out": "姿を消し", "idx": 29563, "score": -12.563262939453125}
+{"in": "verbose", "out": "ことを明らかにした", "idx": 29558, "score": -12.562886238098145}
+{"in": "groups", "out": "その名の通り", "idx": 29551, "score": -12.562413215637207}
+{"in": "attrs", "out": "^^;", "idx": 29547, "score": -12.562051773071289}
+{"in": "size", "out": "にある公立中学校", "idx": 29532, "score": -12.560529708862305}
+{"in": "vals", "out": "とコメントしている", "idx": 29511, "score": -12.557933807373047}
+{"in": "directory", "out": "わかりやすく", "idx": 29506, "score": -12.557350158691406}
+{"in": "parts", "out": "ことはできず", "idx": 29501, "score": -12.557062149047852}
+{"in": "subprocess", "out": "で始まるページの一覧タイトル", "idx": 29485, "score": -12.555493354797363}
+{"in": "ER", "out": "9,000", "idx": 29472, "score": -12.554306983947754}
+{"in": "features", "out": "から派生した", "idx": 29463, "score": -12.552811622619629}
+{"in": "requests", "out": "問題視され", "idx": 29452, "score": -12.551846504211426}
+{"in": "tokens", "out": "えもん", "idx": 29444, "score": -12.550905227661133}
+{"in": "axes", "out": "と推測されている", "idx": 29443, "score": -12.550715446472168}
+{"in": "flags", "out": "の項を参照", "idx": 29431, "score": -12.549659729003906}
+{"in": "samples", "out": "に分布し", "idx": 29421, "score": -12.548849105834961}
+{"in": "apply", "out": "を調査し", "idx": 29398, "score": -12.546494483947754}
+{"in": "ize", "out": "ことが示唆され", "idx": 29393, "score": -12.546011924743652}
+{"in": "IT", "out": "にすぎない", "idx": 29387, "score": -12.5451078414917}
+{"in": "weights", "out": "が適用され", "idx": 29372, "score": -12.543537139892578}
+{"in": "AL", "out": "から解放され", "idx": 29369, "score": -12.543176651000977}
+{"in": "AR", "out": "が死去し", "idx": 29358, "score": -12.54233455657959}
+{"in": "ED", "out": "防止のため", "idx": 29351, "score": -12.540937423706055}
+{"in": "propertie", "out": "なおかつ", "idx": 29347, "score": -12.540244102478027}
+{"in": "ctypes", "out": "たいと思う", "idx": 29331, "score": -12.538861274719238}
+{"in": "ST", "out": "を把握し", "idx": 29328, "score": -12.538528442382812}
+{"in": "AN", "out": "を運営している", "idx": 29319, "score": -12.537568092346191}
+{"in": "attribute", "out": "計画を立て", "idx": 29318, "score": -12.537273406982422}
+{"in": "split", "out": "が発達し", "idx": 29317, "score": -12.537243843078613}
+{"in": "vars", "out": "若い女性", "idx": 29298, "score": -12.535577774047852}
+{"in": "TYPE", "out": "と話した", "idx": 29297, "score": -12.535531997680664}
+{"in": "words", "out": "ことが分かる", "idx": 29288, "score": -12.535032272338867}
+{"in": "Class", "out": "世界で最も", "idx": 29281, "score": -12.534507751464844}
+{"in": "variable", "out": "々しく", "idx": 29279, "score": -12.534379959106445}
+{"in": "coords", "out": "を可能にし", "idx": 29273, "score": -12.534027099609375}
+{"in": "AT", "out": "を操作し", "idx": 29271, "score": -12.533834457397461}
+{"in": "seconds", "out": "ゆうちょ銀行", "idx": 29253, "score": -12.531882286071777}
+{"in": "ify", "out": "ことも多く", "idx": 29234, "score": -12.529809951782227}
+{"in": "parsed", "out": "ことを恐れ", "idx": 29219, "score": -12.528444290161133}
+{"in": "LOG", "out": "を宣言し", "idx": 29213, "score": -12.527778625488281}
+{"in": "Con", "out": "に配慮し", "idx": 29204, "score": -12.526726722717285}
+{"in": "counts", "out": "免許を取得", "idx": 29203, "score": -12.526590347290039}
+{"in": "comp", "out": "年契約で", "idx": 29184, "score": -12.524176597595215}
+{"in": "In", "out": "の悪さ", "idx": 29178, "score": -12.523634910583496}
+{"in": "dumps", "out": "を優先し", "idx": 29166, "score": -12.521889686584473}
+{"in": "obj", "out": "を発射する", "idx": 29161, "score": -12.521334648132324}
+{"in": "inputs", "out": "多くの人々", "idx": 29153, "score": -12.520657539367676}
+{"in": "edges", "out": "月に開催された", "idx": 29147, "score": -12.520158767700195}
+{"in": "policy", "out": "に焦点を当て", "idx": 29143, "score": -12.519235610961914}
+{"in": "units", "out": "2,8", "idx": 29137, "score": -12.519079208374023}
+{"in": "methods", "out": "が予想され", "idx": 29136, "score": -12.519023895263672}
+{"in": "matches", "out": "に入れられ", "idx": 29131, "score": -12.518667221069336}
+{"in": "colors", "out": "を入手し", "idx": 29128, "score": -12.518204689025879}
+{"in": "required", "out": "寒さ", "idx": 29127, "score": -12.518065452575684}
+{"in": "iter", "out": "今のところ", "idx": 29118, "score": -12.517746925354004}
+{"in": "Qt", "out": "ケースもある", "idx": 29101, "score": -12.516353607177734}
+{"in": "property", "out": "ことを認め", "idx": 29092, "score": -12.515559196472168}
+{"in": "Return", "out": "がなされる", "idx": 29086, "score": -12.515178680419922}
+{"in": "loads", "out": "が強化され", "idx": 29079, "score": -12.513934135437012}
+{"in": "alias", "out": "に興味を持ち", "idx": 29069, "score": -12.512958526611328}
+{"in": "fetch", "out": "よく似た", "idx": 29052, "score": -12.51140308380127}
+{"in": "astype", "out": "を構築する", "idx": 29050, "score": -12.51114273071289}
+{"in": "hash", "out": "を前提に", "idx": 29045, "score": -12.510549545288086}
+{"in": "layers", "out": "(^^)", "idx": 29034, "score": -12.509492874145508}
+{"in": "dims", "out": "に感染し", "idx": 29031, "score": -12.509356498718262}
+{"in": "events", "out": "を希望する", "idx": 29029, "score": -12.50919246673584}
+{"in": "contents", "out": "んぐ", "idx": 29026, "score": -12.508731842041016}
+{"in": "actions", "out": "▁4-09-", "idx": 29010, "score": -12.507172584533691}
+{"in": "argument", "out": "活動を再開", "idx": 29001, "score": -12.506525993347168}
+{"in": "lat", "out": "することを決定", "idx": 28986, "score": -12.50487995147705}
+{"in": "elements", "out": "を武器に", "idx": 28979, "score": -12.50408935546875}
+{"in": "ay", "out": "の前に現れ", "idx": 28963, "score": -12.502606391906738}
+{"in": "endswith", "out": "ご自身", "idx": 28958, "score": -12.502312660217285}
+{"in": "filters", "out": "**", "idx": 28950, "score": -12.501532554626465}
+{"in": "ES", "out": "最も有名な", "idx": 28947, "score": -12.501126289367676}
+{"in": "NAME", "out": "▁97840", "idx": 28935, "score": -12.500229835510254}
+{"in": "safe", "out": "を確保するため", "idx": 28934, "score": -12.499862670898438}
+{"in": "exceptions", "out": "を発生させる", "idx": 28920, "score": -12.498503684997559}
+{"in": "format", "out": "恐れがある", "idx": 28913, "score": -12.4978666305542}
+{"in": "summary", "out": "}}{", "idx": 28911, "score": -12.497712135314941}
+{"in": "defaults", "out": "に対抗するため", "idx": 28902, "score": -12.49681568145752}
+{"in": "which", "out": "ご了承ください", "idx": 28865, "score": -12.49402141571045}
+{"in": "OR", "out": "を拠点とする", "idx": 28851, "score": -12.492277145385742}
+{"in": "DEFAULT", "out": "継続的に", "idx": 28843, "score": -12.491694450378418}
+{"in": "warnings", "out": "決勝に進出", "idx": 28841, "score": -12.49164867401123}
+{"in": "states", "out": "を決意する", "idx": 28823, "score": -12.49034309387207}
+{"in": "modules", "out": "事ができ", "idx": 28815, "score": -12.489692687988281}
+{"in": "loss", "out": "が経営する", "idx": 28810, "score": -12.48923397064209}
+{"in": "itertool", "out": "に吸収され", "idx": 28788, "score": -12.486899375915527}
+{"in": "EN", "out": "2,6", "idx": 28786, "score": -12.486804008483887}
+{"in": "My", "out": "の支援を受け", "idx": 28769, "score": -12.485313415527344}
+{"in": "arg", "out": "2,0", "idx": 28735, "score": -12.482708930969238}
+{"in": "Thi", "out": "を与えること", "idx": 28725, "score": -12.482075691223145}
+{"in": "check", "out": "差をつけ", "idx": 28705, "score": -12.480887413024902}
+{"in": "classes", "out": "枠で放送", "idx": 28690, "score": -12.479047775268555}
+{"in": "CLI", "out": "事故が発生", "idx": 28674, "score": -12.477832794189453}
+{"in": "LE", "out": "被害を受け", "idx": 28672, "score": -12.47778606414795}
+{"in": "verify", "out": "見たこと", "idx": 28668, "score": -12.47750186920166}
+PreTrainedTokenizer(name_or_path='sno_py500', vocab_size=32100, model_max_len=1000000000000000019884624838656, is_fast=False, padding_side='right', truncation_side='right', special_tokens={'eos_token': '</s>', 'unk_token': '<unk>', 'pad_token': '<pad>', 'additional_special_tokens': ['<extra_id_0>', '<extra_id_1>', '<extra_id_2>', '<extra_id_3>', '<extra_id_4>', '<extra_id_5>', '<extra_id_6>', '<extra_id_7>', '<extra_id_8>', '<extra_id_9>', '<extra_id_10>', '<extra_id_11>', '<extra_id_12>', '<extra_id_13>', '<extra_id_14>', '<extra_id_15>', '<extra_id_16>', '<extra_id_17>', '<extra_id_18>', '<extra_id_19>', '<extra_id_20>', '<extra_id_21>', '<extra_id_22>', '<extra_id_23>', '<extra_id_24>', '<extra_id_25>', '<extra_id_26>', '<extra_id_27>', '<extra_id_28>', '<extra_id_29>', '<extra_id_30>', '<extra_id_31>', '<extra_id_32>', '<extra_id_33>', '<extra_id_34>', '<extra_id_35>', '<extra_id_36>', '<extra_id_37>', '<extra_id_38>', '<extra_id_39>', '<extra_id_40>', '<extra_id_41>', '<extra_id_42>', '<extra_id_43>', '<extra_id_44>', '<extra_id_45>', '<extra_id_46>', '<extra_id_47>', '<extra_id_48>', '<extra_id_49>', '<extra_id_50>', '<extra_id_51>', '<extra_id_52>', '<extra_id_53>', '<extra_id_54>', '<extra_id_55>', '<extra_id_56>', '<extra_id_57>', '<extra_id_58>', '<extra_id_59>', '<extra_id_60>', '<extra_id_61>', '<extra_id_62>', '<extra_id_63>', '<extra_id_64>', '<extra_id_65>', '<extra_id_66>', '<extra_id_67>', '<extra_id_68>', '<extra_id_69>', '<extra_id_70>', '<extra_id_71>', '<extra_id_72>', '<extra_id_73>', '<extra_id_74>', '<extra_id_75>', '<extra_id_76>', '<extra_id_77>', '<extra_id_78>', '<extra_id_79>', '<extra_id_80>', '<extra_id_81>', '<extra_id_82>', '<extra_id_83>', '<extra_id_84>', '<extra_id_85>', '<extra_id_86>', '<extra_id_87>', '<extra_id_88>', '<extra_id_89>', '<extra_id_90>', '<extra_id_91>', '<extra_id_92>', '<extra_id_93>', '<extra_id_94>', '<extra_id_95>', '<extra_id_96>', '<extra_id_97>', '<extra_id_98>', '<extra_id_99>']})
+<nl><nl> [261, 30635, 30635, 1]
+<123> <100> <1> [30154, 310, 281, 1699, 30154, 701, 1699, 30154, 279, 1699, 1]
+<extra_id_0><extra_id_99> [32099, 32000, 1]
+
+	
+ [1]