nursimakgul commited on
Commit
b297603
·
verified ·
1 Parent(s): 0190e34

Upload backend_tokenizer.vocab with huggingface_hub

Browse files
Files changed (1) hide show
  1. backend_tokenizer.vocab +88 -88
backend_tokenizer.vocab CHANGED
@@ -2,6 +2,7 @@
2
  <unk> 0
3
  <s> 0
4
  </s> 0
 
5
  er -0
6
  in -1
7
  on -2
@@ -1910,91 +1911,90 @@ osoph -1904
1910
  ▁hist -1905
1911
  ▁sept -1906
1912
  cephal -1907
1913
- ologies -1908
1914
- e -1909
1915
- -1910
1916
- i -1911
1917
- a -1912
1918
- n -1913
1919
- o -1914
1920
- r -1915
1921
- s -1916
1922
- t -1917
1923
- l -1918
1924
- c -1919
1925
- u -1920
1926
- d -1921
1927
- p -1922
1928
- m -1923
1929
- h -1924
1930
- g -1925
1931
- y -1926
1932
- b -1927
1933
- f -1928
1934
- - -1929
1935
- v -1930
1936
- k -1931
1937
- w -1932
1938
- z -1933
1939
- x -1934
1940
- S -1935
1941
- C -1936
1942
- A -1937
1943
- q -1938
1944
- M -1939
1945
- P -1940
1946
- j -1941
1947
- B -1942
1948
- T -1943
1949
- L -1944
1950
- ' -1945
1951
- D -1946
1952
- H -1947
1953
- G -1948
1954
- E -1949
1955
- R -1950
1956
- N -1951
1957
- F -1952
1958
- K -1953
1959
- O -1954
1960
- I -1955
1961
- W -1956
1962
- . -1957
1963
- J -1958
1964
- V -1959
1965
- U -1960
1966
- Z -1961
1967
- Y -1962
1968
- / -1963
1969
- Q -1964
1970
- X -1965
1971
- : -1966
1972
- [ -1967
1973
- ] -1968
1974
- ( -1969
1975
- ) -1970
1976
- 2 -1971
1977
- 1 -1972
1978
- 0 -1973
1979
- = -1974
1980
- , -1975
1981
- # -1976
1982
- 3 -1977
1983
- 4 -1978
1984
- ? -1979
1985
- " -1980
1986
- ! -1981
1987
- & -1982
1988
- 5 -1983
1989
- 6 -1984
1990
- 8 -1985
1991
- ` -1986
1992
- 9 -1987
1993
- % -1988
1994
- @ -1989
1995
- 7 -1990
1996
- < -1991
1997
- > -1992
1998
- $ -1993
1999
- + -1994
2000
- ; -1995
 
2
  <unk> 0
3
  <s> 0
4
  </s> 0
5
+ <mask> 0
6
  er -0
7
  in -1
8
  on -2
 
1911
  ▁hist -1905
1912
  ▁sept -1906
1913
  cephal -1907
1914
+ e -1908
1915
+ -1909
1916
+ i -1910
1917
+ a -1911
1918
+ n -1912
1919
+ o -1913
1920
+ r -1914
1921
+ s -1915
1922
+ t -1916
1923
+ l -1917
1924
+ c -1918
1925
+ u -1919
1926
+ d -1920
1927
+ p -1921
1928
+ m -1922
1929
+ h -1923
1930
+ g -1924
1931
+ y -1925
1932
+ b -1926
1933
+ f -1927
1934
+ - -1928
1935
+ v -1929
1936
+ k -1930
1937
+ w -1931
1938
+ z -1932
1939
+ x -1933
1940
+ S -1934
1941
+ C -1935
1942
+ A -1936
1943
+ q -1937
1944
+ M -1938
1945
+ P -1939
1946
+ j -1940
1947
+ B -1941
1948
+ T -1942
1949
+ L -1943
1950
+ ' -1944
1951
+ D -1945
1952
+ H -1946
1953
+ G -1947
1954
+ E -1948
1955
+ R -1949
1956
+ N -1950
1957
+ F -1951
1958
+ K -1952
1959
+ O -1953
1960
+ I -1954
1961
+ W -1955
1962
+ . -1956
1963
+ J -1957
1964
+ V -1958
1965
+ U -1959
1966
+ Z -1960
1967
+ Y -1961
1968
+ / -1962
1969
+ Q -1963
1970
+ X -1964
1971
+ : -1965
1972
+ [ -1966
1973
+ ] -1967
1974
+ ( -1968
1975
+ ) -1969
1976
+ 2 -1970
1977
+ 1 -1971
1978
+ 0 -1972
1979
+ = -1973
1980
+ , -1974
1981
+ # -1975
1982
+ 3 -1976
1983
+ 4 -1977
1984
+ ? -1978
1985
+ " -1979
1986
+ ! -1980
1987
+ & -1981
1988
+ 5 -1982
1989
+ 6 -1983
1990
+ 8 -1984
1991
+ ` -1985
1992
+ 9 -1986
1993
+ % -1987
1994
+ @ -1988
1995
+ 7 -1989
1996
+ < -1990
1997
+ > -1991
1998
+ $ -1992
1999
+ + -1993
2000
+ ; -1994