Rizqi commited on
Commit
2e7eba3
·
1 Parent(s): 671f972

Create sequence

Browse files
Files changed (1) hide show
  1. sequence +9 -0
sequence ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ token_lens = []
2
+
3
+ for txt in data.Text:
4
+ tokens = bert_tokenizer.encode(txt)
5
+ token_lens.append(len(tokens))
6
+
7
+ sns.histplot(token_lens, kde=True, stat='density', linewidth=0)
8
+ plt.xlim([0, 100]);
9
+ plt.xlabel('Token count');