raptorkwok
/

pyctokenizer

Model card Files Files and versions

raptorkwok commited on Jan 20

Commit

ae251d4

·

verified ·

1 Parent(s): 78f34dd

Upload tokenizer

Files changed (2) hide show

.gitattributes +1 -0
tokenizer.json +3 -0

.gitattributes CHANGED Viewed

@@ -34,3 +34,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 tokenizer_config.json filter=lfs diff=lfs merge=lfs -text

 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 tokenizer_config.json filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

tokenizer.json ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:1eaa34779bc58f2145faf659028d6e0e76a051bd31c33ee1ac2704855e8fbd5b
+size 21356002