Yuchan commited on
Commit
ec2770f
·
verified ·
1 Parent(s): 799f458

Update Mo.py

Browse files
Files changed (1) hide show
  1. Mo.py +3 -1
Mo.py CHANGED
@@ -99,6 +99,8 @@ def txt_stream(file_path):
99
  )
100
 
101
 
 
 
102
  dataset = tf.data.Dataset.from_generator(
103
  lambda: txt_stream(DATA_PATH),
104
  output_signature=(
@@ -107,7 +109,7 @@ dataset = tf.data.Dataset.from_generator(
107
  )
108
  )
109
 
110
- dataset = dataset.shuffle(2000, seed=SEED).batch(batch_size, drop_remainder=True).prefetch(tf.data.AUTOTUNE)
111
 
112
  with strategy.scope():
113
  dist_dataset = strategy.experimental_distribute_dataset(dataset)
 
99
  )
100
 
101
 
102
+ LIMIT = 500000 # 원하는 만큼
103
+
104
  dataset = tf.data.Dataset.from_generator(
105
  lambda: txt_stream(DATA_PATH),
106
  output_signature=(
 
109
  )
110
  )
111
 
112
+ dataset = dataset.take(LIMIT).shuffle(2000, seed=SEED).batch(batch_size, drop_remainder=True).prefetch(tf.data.AUTOTUNE)
113
 
114
  with strategy.scope():
115
  dist_dataset = strategy.experimental_distribute_dataset(dataset)