Hugging Face's logo

Coding-With-Bashir
/
bwenge_ai

Model card Files
xet
 Community
Coding-With-Bashir commited on
Commit
dce1937
·
verified ·
1 Parent(s): 580c316

Upload .\data\processed\processing_summary.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. .//data//processed//processing_summary.json +266 -0
.//data//processed//processing_summary.json ADDED
@@ -0,0 +1,266 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "total_loaded": 1239240,
3
+ "total_after_lang_filter": 755703,
4
+ "total_after_dedup": 661014,
5
+ "total_tokens": 24704887,
6
+ "total_training_samples": 661008,
7
+ "total_instruction": 95437,
8
+ "total_chat": 0,
9
+ "per_file_stats": [
10
+ {
11
+ "file": "huggingface_ChrisToukmaji_kinyarwanda_instruction_tuning.jsonl",
12
+ "loaded": 4250,
13
+ "lang_filtered": 2,
14
+ "clean_filtered": 0,
15
+ "kept": 4248,
16
+ "tokens": 327390,
17
+ "instruction_samples": 4248,
18
+ "chat_samples": 0
19
+ },
20
+ {
21
+ "file": "huggingface_CircuitNotion_kinyarwanda_corpus.jsonl",
22
+ "loaded": 0,
23
+ "lang_filtered": 0,
24
+ "clean_filtered": 0,
25
+ "kept": 0,
26
+ "tokens": 0,
27
+ "instruction_samples": 0,
28
+ "chat_samples": 0
29
+ },
30
+ {
31
+ "file": "huggingface_DigitalUmuganda_kinyarwanda-english-machine-translation-dataset.jsonl",
32
+ "loaded": 0,
33
+ "lang_filtered": 0,
34
+ "clean_filtered": 0,
35
+ "kept": 0,
36
+ "tokens": 0,
37
+ "instruction_samples": 0,
38
+ "chat_samples": 0
39
+ },
40
+ {
41
+ "file": "huggingface_DigitalUmuganda_NMT_Health_parallel_data_en_kin.jsonl",
42
+ "loaded": 30000,
43
+ "lang_filtered": 1,
44
+ "clean_filtered": 4274,
45
+ "kept": 25725,
46
+ "tokens": 524882,
47
+ "instruction_samples": 0,
48
+ "chat_samples": 0
49
+ },
50
+ {
51
+ "file": "huggingface_DigitalUmuganda_NMT_Rwandan-Gazette_parallel_data_en_kin.jsonl",
52
+ "loaded": 80000,
53
+ "lang_filtered": 0,
54
+ "clean_filtered": 9768,
55
+ "kept": 70232,
56
+ "tokens": 1727805,
57
+ "instruction_samples": 0,
58
+ "chat_samples": 0
59
+ },
60
+ {
61
+ "file": "huggingface_jojo-ai-mst_Roleplay-Kinyarwanda.jsonl",
62
+ "loaded": 1923,
63
+ "lang_filtered": 0,
64
+ "clean_filtered": 0,
65
+ "kept": 1923,
66
+ "tokens": 50283,
67
+ "instruction_samples": 1923,
68
+ "chat_samples": 0
69
+ },
70
+ {
71
+ "file": "huggingface_mbazaNLP_Kinyarwanda_English_parallel_dataset.jsonl",
72
+ "loaded": 55667,
73
+ "lang_filtered": 0,
74
+ "clean_filtered": 7384,
75
+ "kept": 48283,
76
+ "tokens": 779663,
77
+ "instruction_samples": 0,
78
+ "chat_samples": 0
79
+ },
80
+ {
81
+ "file": "huggingface_michsethowusu_akan-kinyarwanda_sentence-pairs.jsonl",
82
+ "loaded": 67145,
83
+ "lang_filtered": 7800,
84
+ "clean_filtered": 39704,
85
+ "kept": 19641,
86
+ "tokens": 294505,
87
+ "instruction_samples": 0,
88
+ "chat_samples": 0
89
+ },
90
+ {
91
+ "file": "huggingface_michsethowusu_bambara-kinyarwanda_sentence-pairs.jsonl",
92
+ "loaded": 56301,
93
+ "lang_filtered": 3366,
94
+ "clean_filtered": 25937,
95
+ "kept": 26998,
96
+ "tokens": 522282,
97
+ "instruction_samples": 0,
98
+ "chat_samples": 0
99
+ },
100
+ {
101
+ "file": "huggingface_michsethowusu_bemba-kinyarwanda_sentence-pairs.jsonl",
102
+ "loaded": 197886,
103
+ "lang_filtered": 2,
104
+ "clean_filtered": 101878,
105
+ "kept": 96006,
106
+ "tokens": 1379707,
107
+ "instruction_samples": 0,
108
+ "chat_samples": 0
109
+ },
110
+ {
111
+ "file": "huggingface_michsethowusu_dinka-kinyarwanda_sentence-pairs.jsonl",
112
+ "loaded": 39196,
113
+ "lang_filtered": 6663,
114
+ "clean_filtered": 18172,
115
+ "kept": 14361,
116
+ "tokens": 268754,
117
+ "instruction_samples": 0,
118
+ "chat_samples": 0
119
+ },
120
+ {
121
+ "file": "huggingface_michsethowusu_dyula-kinyarwanda_sentence-pairs.jsonl",
122
+ "loaded": 102925,
123
+ "lang_filtered": 2446,
124
+ "clean_filtered": 57811,
125
+ "kept": 42668,
126
+ "tokens": 857116,
127
+ "instruction_samples": 0,
128
+ "chat_samples": 0
129
+ },
130
+ {
131
+ "file": "huggingface_michsethowusu_kinyarwanda-rundi_sentence-pairs.jsonl",
132
+ "loaded": 410000,
133
+ "lang_filtered": 11,
134
+ "clean_filtered": 163087,
135
+ "kept": 246902,
136
+ "tokens": 3516163,
137
+ "instruction_samples": 0,
138
+ "chat_samples": 0
139
+ },
140
+ {
141
+ "file": "huggingface_Mikecyane_Kinyarwanda_chat.jsonl",
142
+ "loaded": 0,
143
+ "lang_filtered": 0,
144
+ "clean_filtered": 0,
145
+ "kept": 0,
146
+ "tokens": 0,
147
+ "instruction_samples": 0,
148
+ "chat_samples": 0
149
+ },
150
+ {
151
+ "file": "huggingface_RogerB_kin_en_DigitalUmuganda.jsonl",
152
+ "loaded": 47824,
153
+ "lang_filtered": 0,
154
+ "clean_filtered": 30920,
155
+ "kept": 16904,
156
+ "tokens": 159881,
157
+ "instruction_samples": 0,
158
+ "chat_samples": 0
159
+ },
160
+ {
161
+ "file": "huggingface_saillab_alpaca-kinyarwanda-cleaned.jsonl",
162
+ "loaded": 41601,
163
+ "lang_filtered": 2,
164
+ "clean_filtered": 1900,
165
+ "kept": 39699,
166
+ "tokens": 1440835,
167
+ "instruction_samples": 41599,
168
+ "chat_samples": 0
169
+ },
170
+ {
171
+ "file": "huggingface_saillab_alpaca_kinyarwanda_taco.jsonl",
172
+ "loaded": 49601,
173
+ "lang_filtered": 11,
174
+ "clean_filtered": 0,
175
+ "kept": 49590,
176
+ "tokens": 12116090,
177
+ "instruction_samples": 49590,
178
+ "chat_samples": 0
179
+ },
180
+ {
181
+ "file": "huggingface_souvorinkg_english_kinyarwanda.jsonl",
182
+ "loaded": 54589,
183
+ "lang_filtered": 0,
184
+ "clean_filtered": 2387,
185
+ "kept": 52202,
186
+ "tokens": 662665,
187
+ "instruction_samples": 0,
188
+ "chat_samples": 0
189
+ },
190
+ {
191
+ "file": "igihe_igihe_articles.jsonl",
192
+ "loaded": 0,
193
+ "lang_filtered": 0,
194
+ "clean_filtered": 0,
195
+ "kept": 0,
196
+ "tokens": 0,
197
+ "instruction_samples": 0,
198
+ "chat_samples": 0
199
+ },
200
+ {
201
+ "file": "kigalitoday_kigalitoday_articles.jsonl",
202
+ "loaded": 0,
203
+ "lang_filtered": 0,
204
+ "clean_filtered": 0,
205
+ "kept": 0,
206
+ "tokens": 0,
207
+ "instruction_samples": 0,
208
+ "chat_samples": 0
209
+ },
210
+ {
211
+ "file": "newtimes_rss_newtimes_rss_articles.jsonl",
212
+ "loaded": 0,
213
+ "lang_filtered": 0,
214
+ "clean_filtered": 0,
215
+ "kept": 0,
216
+ "tokens": 0,
217
+ "instruction_samples": 0,
218
+ "chat_samples": 0
219
+ },
220
+ {
221
+ "file": "rbc_rbc_articles.jsonl",
222
+ "loaded": 0,
223
+ "lang_filtered": 0,
224
+ "clean_filtered": 0,
225
+ "kept": 0,
226
+ "tokens": 0,
227
+ "instruction_samples": 0,
228
+ "chat_samples": 0
229
+ },
230
+ {
231
+ "file": "wikipedia_wikipedia_articles.jsonl",
232
+ "loaded": 274,
233
+ "lang_filtered": 0,
234
+ "clean_filtered": 0,
235
+ "kept": 274,
236
+ "tokens": 75991,
237
+ "instruction_samples": 0,
238
+ "chat_samples": 0
239
+ },
240
+ {
241
+ "file": "wikisource_wikisource_articles.jsonl",
242
+ "loaded": 0,
243
+ "lang_filtered": 0,
244
+ "clean_filtered": 0,
245
+ "kept": 0,
246
+ "tokens": 0,
247
+ "instruction_samples": 0,
248
+ "chat_samples": 0
249
+ },
250
+ {
251
+ "file": "wiktionary_wiktionary_entries.jsonl",
252
+ "loaded": 58,
253
+ "lang_filtered": 11,
254
+ "clean_filtered": 0,
255
+ "kept": 47,
256
+ "tokens": 875,
257
+ "instruction_samples": 0,
258
+ "chat_samples": 0
259
+ }
260
+ ],
261
+ "output_files": {
262
+ "training": "data\\processed\\training_data.jsonl",
263
+ "instruction": "data\\processed\\instruction_data.jsonl",
264
+ "chat": "data\\processed\\chat_data.jsonl"
265
+ }
266
+ }