fajarluhung commited on
Commit
2bf6c9c
·
1 Parent(s): 9e09adf

Training done

Browse files
Files changed (3) hide show
  1. added_tokens.json +1 -0
  2. preprocessor_config.json +2 -2
  3. tokenizer.json +11 -16
added_tokens.json CHANGED
@@ -2,6 +2,7 @@
2
  "</s_agama>": 57548,
3
  "</s_alamat>": 57540,
4
  "</s_berlaku>": 57556,
 
5
  "</s_desa>": 57544,
6
  "</s_goldar>": 57538,
7
  "</s_kabkot>": 57528,
 
2
  "</s_agama>": 57548,
3
  "</s_alamat>": 57540,
4
  "</s_berlaku>": 57556,
5
+ "</s_data>": 57558,
6
  "</s_desa>": 57544,
7
  "</s_goldar>": 57538,
8
  "</s_kabkot>": 57528,
preprocessor_config.json CHANGED
@@ -20,7 +20,7 @@
20
  "resample": 2,
21
  "rescale_factor": 0.00392156862745098,
22
  "size": [
23
- 960,
24
- 1280
25
  ]
26
  }
 
20
  "resample": 2,
21
  "rescale_factor": 0.00392156862745098,
22
  "size": [
23
+ 1280,
24
+ 960
25
  ]
26
  }
tokenizer.json CHANGED
@@ -1,21 +1,7 @@
1
  {
2
  "version": "1.0",
3
- "truncation": {
4
- "direction": "Right",
5
- "max_length": 768,
6
- "strategy": "LongestFirst",
7
- "stride": 0
8
- },
9
- "padding": {
10
- "strategy": {
11
- "Fixed": 768
12
- },
13
- "direction": "Right",
14
- "pad_to_multiple_of": null,
15
- "pad_id": 1,
16
- "pad_type_id": 0,
17
- "pad_token": "<pad>"
18
- },
19
  "added_tokens": [
20
  {
21
  "id": 0,
@@ -385,6 +371,15 @@
385
  "rstrip": false,
386
  "normalized": true,
387
  "special": false
 
 
 
 
 
 
 
 
 
388
  }
389
  ],
390
  "normalizer": {
 
1
  {
2
  "version": "1.0",
3
+ "truncation": null,
4
+ "padding": null,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5
  "added_tokens": [
6
  {
7
  "id": 0,
 
371
  "rstrip": false,
372
  "normalized": true,
373
  "special": false
374
+ },
375
+ {
376
+ "id": 57558,
377
+ "content": "</s_data>",
378
+ "single_word": false,
379
+ "lstrip": false,
380
+ "rstrip": false,
381
+ "normalized": true,
382
+ "special": false
383
  }
384
  ],
385
  "normalizer": {