Spaces:
Build error
Build error
PeteBleackley commited on
Commit ·
b2ffb12
1
Parent(s): cfddf8c
Reduced batch size
Browse files
qarac/corpora/CombinedCorpus.py
CHANGED
|
@@ -37,7 +37,7 @@ class CombinedCorpus(torch.utils.data.IterableDataset):
|
|
| 37 |
'encode_decode')})
|
| 38 |
n_samples = len(self.all_text)
|
| 39 |
|
| 40 |
-
self.n_batches = n_samples//
|
| 41 |
self.question_answering = CorpusRepeater.CorpusRepeater(CorpusLoader.CorpusLoader(kwargs['question_answering'],
|
| 42 |
tokenizer,
|
| 43 |
['question',
|
|
@@ -110,7 +110,7 @@ class CombinedCorpus(torch.utils.data.IterableDataset):
|
|
| 110 |
for sample in self.samples():
|
| 111 |
batch.append(sample)
|
| 112 |
n+=1
|
| 113 |
-
if n==
|
| 114 |
yield(self.batch(batch))
|
| 115 |
batch = []
|
| 116 |
n=0
|
|
|
|
| 37 |
'encode_decode')})
|
| 38 |
n_samples = len(self.all_text)
|
| 39 |
|
| 40 |
+
self.n_batches = n_samples//16
|
| 41 |
self.question_answering = CorpusRepeater.CorpusRepeater(CorpusLoader.CorpusLoader(kwargs['question_answering'],
|
| 42 |
tokenizer,
|
| 43 |
['question',
|
|
|
|
| 110 |
for sample in self.samples():
|
| 111 |
batch.append(sample)
|
| 112 |
n+=1
|
| 113 |
+
if n==16:
|
| 114 |
yield(self.batch(batch))
|
| 115 |
batch = []
|
| 116 |
n=0
|