PeteBleackley commited on
Commit
b2ffb12
·
1 Parent(s): cfddf8c

Reduced batch size

Browse files
Files changed (1) hide show
  1. qarac/corpora/CombinedCorpus.py +2 -2
qarac/corpora/CombinedCorpus.py CHANGED
@@ -37,7 +37,7 @@ class CombinedCorpus(torch.utils.data.IterableDataset):
37
  'encode_decode')})
38
  n_samples = len(self.all_text)
39
 
40
- self.n_batches = n_samples//32
41
  self.question_answering = CorpusRepeater.CorpusRepeater(CorpusLoader.CorpusLoader(kwargs['question_answering'],
42
  tokenizer,
43
  ['question',
@@ -110,7 +110,7 @@ class CombinedCorpus(torch.utils.data.IterableDataset):
110
  for sample in self.samples():
111
  batch.append(sample)
112
  n+=1
113
- if n==32:
114
  yield(self.batch(batch))
115
  batch = []
116
  n=0
 
37
  'encode_decode')})
38
  n_samples = len(self.all_text)
39
 
40
+ self.n_batches = n_samples//16
41
  self.question_answering = CorpusRepeater.CorpusRepeater(CorpusLoader.CorpusLoader(kwargs['question_answering'],
42
  tokenizer,
43
  ['question',
 
110
  for sample in self.samples():
111
  batch.append(sample)
112
  n+=1
113
+ if n==16:
114
  yield(self.batch(batch))
115
  batch = []
116
  n=0