PeteBleackley commited on
Commit
cfddf8c
·
1 Parent(s): 432a965

Google Colab imports

Browse files
Files changed (1) hide show
  1. scripts.py +4 -4
scripts.py CHANGED
@@ -20,7 +20,7 @@ import seaborn
20
  import tqdm
21
  import gradio
22
  import boto3
23
- import google.colab
24
 
25
  class SequenceCrossEntropyLoss(torch.nn.Module):
26
  def __init__(self):
@@ -61,8 +61,8 @@ def download_training_data():
61
  if not os.path.exists('corpora'):
62
  os.makedirs('corpora')
63
  s3 = boto3.client('s3',
64
- aws_access_key_id=google.colab.userdata.get('AWS_KEY'),
65
- aws_secret_access_key=google.colab.userdata.get('AWS_SECRET'))
66
  for obj in s3.list_objects(Bucket='qarac')['Contents']:
67
  filename = obj['Key']
68
  s3.download_file('qarac',filename,'corpora/{}'.format(filename))
@@ -155,7 +155,7 @@ def train_models(path,progress=gradio.Progress(track_tqdm=True)):
155
  epoch_data[batch] = loss.item()
156
  history[epoch_label] = epoch_data
157
  scheduler.step()
158
- huggingface_hub.login(token=google.colab.userdata.get('HUGGINGFACE_TOKEN'))
159
  trainer.question_encoder.push_to_hub('{}/qarac-roberta-question-encoder'.format(path))
160
  trainer.answer_encoder.push_to_hub('{}/qarac-roberta-answer-encoder'.format(path))
161
  trainer.decoder.push_to_hub('{}/qarac-roberta-decoder'.format(path))
 
20
  import tqdm
21
  import gradio
22
  import boto3
23
+ from google.colab import userdata
24
 
25
  class SequenceCrossEntropyLoss(torch.nn.Module):
26
  def __init__(self):
 
61
  if not os.path.exists('corpora'):
62
  os.makedirs('corpora')
63
  s3 = boto3.client('s3',
64
+ aws_access_key_id=userdata.get('AWS_KEY'),
65
+ aws_secret_access_key=userdata.get('AWS_SECRET'))
66
  for obj in s3.list_objects(Bucket='qarac')['Contents']:
67
  filename = obj['Key']
68
  s3.download_file('qarac',filename,'corpora/{}'.format(filename))
 
155
  epoch_data[batch] = loss.item()
156
  history[epoch_label] = epoch_data
157
  scheduler.step()
158
+ huggingface_hub.login(token=userdata.get('HUGGINGFACE_TOKEN'))
159
  trainer.question_encoder.push_to_hub('{}/qarac-roberta-question-encoder'.format(path))
160
  trainer.answer_encoder.push_to_hub('{}/qarac-roberta-answer-encoder'.format(path))
161
  trainer.decoder.push_to_hub('{}/qarac-roberta-decoder'.format(path))