Hugging Face's logo

yahyaabd
/
sbert-bps-custom-tokenizer

Sentence Similarity
sentence-transformers
Safetensors
bert
feature-extraction
text-embeddings-inference
Model card Files
xet
Community
yahyaabd commited on
Commit
90a284f
·
verified ·
1 Parent(s): 6241aba

Upload folder using huggingface_hub

Browse files
Files changed (5) hide show
  1. added_tokens.json +34 -1
  2. config.json +1 -1
  3. model.safetensors +2 -2
  4. tokenizer.json +2 -2
  5. tokenizer_config.json +264 -0
added_tokens.json CHANGED
@@ -40,5 +40,38 @@
40
  "Statistik Sektoral": 250041,
41
  "TPAK": 250033,
42
  "TPK": 250011,
43
- "TPT": 250012
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
44
  }
 
40
  "Statistik Sektoral": 250041,
41
  "TPAK": 250033,
42
  "TPK": 250011,
43
+ "TPT": 250012,
44
+ "aceh": 250044,
45
+ "babel": 250052,
46
+ "banten": 250059,
47
+ "bengkulu": 250050,
48
+ "gorontalo": 250071,
49
+ "jabar": 250055,
50
+ "jakarta": 250054,
51
+ "jambi": 250049,
52
+ "jateng": 250056,
53
+ "jatim": 250057,
54
+ "kalbar": 250062,
55
+ "kalsel": 250064,
56
+ "kalteng": 250063,
57
+ "kaltim": 250065,
58
+ "kalut": 250066,
59
+ "kepri": 250048,
60
+ "lampung": 250053,
61
+ "maluku": 250073,
62
+ "malut": 250074,
63
+ "ntb": 250060,
64
+ "ntt": 250061,
65
+ "papbar": 250076,
66
+ "papua": 250075,
67
+ "riau": 250047,
68
+ "sulbar": 250072,
69
+ "sulsel": 250069,
70
+ "sulteng": 250068,
71
+ "sultra": 250070,
72
+ "sulut": 250067,
73
+ "sumbar": 250046,
74
+ "sumsel": 250051,
75
+ "sumut": 250045,
76
+ "yogyakarta": 250058
77
  }
config.json CHANGED
@@ -21,5 +21,5 @@
21
  "transformers_version": "4.51.3",
22
  "type_vocab_size": 2,
23
  "use_cache": true,
24
- "vocab_size": 250044
25
  }
 
21
  "transformers_version": "4.51.3",
22
  "type_vocab_size": 2,
23
  "use_cache": true,
24
+ "vocab_size": 250077
25
  }
model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:6b4bd75e54ee056669b6b07ae9af25a63b94fd2db77b477d598573dfea96d246
3
- size 470648168
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:43ea168220bf98fe67f328d47ff1b26c1dbd1fd8a00bc49a2e0804828ab38249
3
+ size 470698856
tokenizer.json CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:e8c3177dc98169def07b03c826f12c91dd7dbd8e48e0affcf8e6fce6326df5e4
3
- size 17090609
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2ac6325d5b609b2e268a86a31b12051a277b12d23e5fee361da74c8dd09351c7
3
+ size 17096641
tokenizer_config.json CHANGED
@@ -375,6 +375,270 @@
375
  "rstrip": false,
376
  "single_word": false,
377
  "special": false
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
378
  }
379
  },
380
  "bos_token": "<s>",
 
375
  "rstrip": false,
376
  "single_word": false,
377
  "special": false
378
+ },
379
+ "250044": {
380
+ "content": "aceh",
381
+ "lstrip": false,
382
+ "normalized": true,
383
+ "rstrip": false,
384
+ "single_word": false,
385
+ "special": false
386
+ },
387
+ "250045": {
388
+ "content": "sumut",
389
+ "lstrip": false,
390
+ "normalized": true,
391
+ "rstrip": false,
392
+ "single_word": false,
393
+ "special": false
394
+ },
395
+ "250046": {
396
+ "content": "sumbar",
397
+ "lstrip": false,
398
+ "normalized": true,
399
+ "rstrip": false,
400
+ "single_word": false,
401
+ "special": false
402
+ },
403
+ "250047": {
404
+ "content": "riau",
405
+ "lstrip": false,
406
+ "normalized": true,
407
+ "rstrip": false,
408
+ "single_word": false,
409
+ "special": false
410
+ },
411
+ "250048": {
412
+ "content": "kepri",
413
+ "lstrip": false,
414
+ "normalized": true,
415
+ "rstrip": false,
416
+ "single_word": false,
417
+ "special": false
418
+ },
419
+ "250049": {
420
+ "content": "jambi",
421
+ "lstrip": false,
422
+ "normalized": true,
423
+ "rstrip": false,
424
+ "single_word": false,
425
+ "special": false
426
+ },
427
+ "250050": {
428
+ "content": "bengkulu",
429
+ "lstrip": false,
430
+ "normalized": true,
431
+ "rstrip": false,
432
+ "single_word": false,
433
+ "special": false
434
+ },
435
+ "250051": {
436
+ "content": "sumsel",
437
+ "lstrip": false,
438
+ "normalized": true,
439
+ "rstrip": false,
440
+ "single_word": false,
441
+ "special": false
442
+ },
443
+ "250052": {
444
+ "content": "babel",
445
+ "lstrip": false,
446
+ "normalized": true,
447
+ "rstrip": false,
448
+ "single_word": false,
449
+ "special": false
450
+ },
451
+ "250053": {
452
+ "content": "lampung",
453
+ "lstrip": false,
454
+ "normalized": true,
455
+ "rstrip": false,
456
+ "single_word": false,
457
+ "special": false
458
+ },
459
+ "250054": {
460
+ "content": "jakarta",
461
+ "lstrip": false,
462
+ "normalized": true,
463
+ "rstrip": false,
464
+ "single_word": false,
465
+ "special": false
466
+ },
467
+ "250055": {
468
+ "content": "jabar",
469
+ "lstrip": false,
470
+ "normalized": true,
471
+ "rstrip": false,
472
+ "single_word": false,
473
+ "special": false
474
+ },
475
+ "250056": {
476
+ "content": "jateng",
477
+ "lstrip": false,
478
+ "normalized": true,
479
+ "rstrip": false,
480
+ "single_word": false,
481
+ "special": false
482
+ },
483
+ "250057": {
484
+ "content": "jatim",
485
+ "lstrip": false,
486
+ "normalized": true,
487
+ "rstrip": false,
488
+ "single_word": false,
489
+ "special": false
490
+ },
491
+ "250058": {
492
+ "content": "yogyakarta",
493
+ "lstrip": false,
494
+ "normalized": true,
495
+ "rstrip": false,
496
+ "single_word": false,
497
+ "special": false
498
+ },
499
+ "250059": {
500
+ "content": "banten",
501
+ "lstrip": false,
502
+ "normalized": true,
503
+ "rstrip": false,
504
+ "single_word": false,
505
+ "special": false
506
+ },
507
+ "250060": {
508
+ "content": "ntb",
509
+ "lstrip": false,
510
+ "normalized": true,
511
+ "rstrip": false,
512
+ "single_word": false,
513
+ "special": false
514
+ },
515
+ "250061": {
516
+ "content": "ntt",
517
+ "lstrip": false,
518
+ "normalized": true,
519
+ "rstrip": false,
520
+ "single_word": false,
521
+ "special": false
522
+ },
523
+ "250062": {
524
+ "content": "kalbar",
525
+ "lstrip": false,
526
+ "normalized": true,
527
+ "rstrip": false,
528
+ "single_word": false,
529
+ "special": false
530
+ },
531
+ "250063": {
532
+ "content": "kalteng",
533
+ "lstrip": false,
534
+ "normalized": true,
535
+ "rstrip": false,
536
+ "single_word": false,
537
+ "special": false
538
+ },
539
+ "250064": {
540
+ "content": "kalsel",
541
+ "lstrip": false,
542
+ "normalized": true,
543
+ "rstrip": false,
544
+ "single_word": false,
545
+ "special": false
546
+ },
547
+ "250065": {
548
+ "content": "kaltim",
549
+ "lstrip": false,
550
+ "normalized": true,
551
+ "rstrip": false,
552
+ "single_word": false,
553
+ "special": false
554
+ },
555
+ "250066": {
556
+ "content": "kalut",
557
+ "lstrip": false,
558
+ "normalized": true,
559
+ "rstrip": false,
560
+ "single_word": false,
561
+ "special": false
562
+ },
563
+ "250067": {
564
+ "content": "sulut",
565
+ "lstrip": false,
566
+ "normalized": true,
567
+ "rstrip": false,
568
+ "single_word": false,
569
+ "special": false
570
+ },
571
+ "250068": {
572
+ "content": "sulteng",
573
+ "lstrip": false,
574
+ "normalized": true,
575
+ "rstrip": false,
576
+ "single_word": false,
577
+ "special": false
578
+ },
579
+ "250069": {
580
+ "content": "sulsel",
581
+ "lstrip": false,
582
+ "normalized": true,
583
+ "rstrip": false,
584
+ "single_word": false,
585
+ "special": false
586
+ },
587
+ "250070": {
588
+ "content": "sultra",
589
+ "lstrip": false,
590
+ "normalized": true,
591
+ "rstrip": false,
592
+ "single_word": false,
593
+ "special": false
594
+ },
595
+ "250071": {
596
+ "content": "gorontalo",
597
+ "lstrip": false,
598
+ "normalized": true,
599
+ "rstrip": false,
600
+ "single_word": false,
601
+ "special": false
602
+ },
603
+ "250072": {
604
+ "content": "sulbar",
605
+ "lstrip": false,
606
+ "normalized": true,
607
+ "rstrip": false,
608
+ "single_word": false,
609
+ "special": false
610
+ },
611
+ "250073": {
612
+ "content": "maluku",
613
+ "lstrip": false,
614
+ "normalized": true,
615
+ "rstrip": false,
616
+ "single_word": false,
617
+ "special": false
618
+ },
619
+ "250074": {
620
+ "content": "malut",
621
+ "lstrip": false,
622
+ "normalized": true,
623
+ "rstrip": false,
624
+ "single_word": false,
625
+ "special": false
626
+ },
627
+ "250075": {
628
+ "content": "papua",
629
+ "lstrip": false,
630
+ "normalized": true,
631
+ "rstrip": false,
632
+ "single_word": false,
633
+ "special": false
634
+ },
635
+ "250076": {
636
+ "content": "papbar",
637
+ "lstrip": false,
638
+ "normalized": true,
639
+ "rstrip": false,
640
+ "single_word": false,
641
+ "special": false
642
  }
643
  },
644
  "bos_token": "<s>",