Spaces:
Build error
Build error
PeteBleackley
commited on
Commit
·
cfddf8c
1
Parent(s):
432a965
Google Colab imports
Browse files- scripts.py +4 -4
scripts.py
CHANGED
|
@@ -20,7 +20,7 @@ import seaborn
|
|
| 20 |
import tqdm
|
| 21 |
import gradio
|
| 22 |
import boto3
|
| 23 |
-
|
| 24 |
|
| 25 |
class SequenceCrossEntropyLoss(torch.nn.Module):
|
| 26 |
def __init__(self):
|
|
@@ -61,8 +61,8 @@ def download_training_data():
|
|
| 61 |
if not os.path.exists('corpora'):
|
| 62 |
os.makedirs('corpora')
|
| 63 |
s3 = boto3.client('s3',
|
| 64 |
-
aws_access_key_id=
|
| 65 |
-
aws_secret_access_key=
|
| 66 |
for obj in s3.list_objects(Bucket='qarac')['Contents']:
|
| 67 |
filename = obj['Key']
|
| 68 |
s3.download_file('qarac',filename,'corpora/{}'.format(filename))
|
|
@@ -155,7 +155,7 @@ def train_models(path,progress=gradio.Progress(track_tqdm=True)):
|
|
| 155 |
epoch_data[batch] = loss.item()
|
| 156 |
history[epoch_label] = epoch_data
|
| 157 |
scheduler.step()
|
| 158 |
-
huggingface_hub.login(token=
|
| 159 |
trainer.question_encoder.push_to_hub('{}/qarac-roberta-question-encoder'.format(path))
|
| 160 |
trainer.answer_encoder.push_to_hub('{}/qarac-roberta-answer-encoder'.format(path))
|
| 161 |
trainer.decoder.push_to_hub('{}/qarac-roberta-decoder'.format(path))
|
|
|
|
| 20 |
import tqdm
|
| 21 |
import gradio
|
| 22 |
import boto3
|
| 23 |
+
from google.colab import userdata
|
| 24 |
|
| 25 |
class SequenceCrossEntropyLoss(torch.nn.Module):
|
| 26 |
def __init__(self):
|
|
|
|
| 61 |
if not os.path.exists('corpora'):
|
| 62 |
os.makedirs('corpora')
|
| 63 |
s3 = boto3.client('s3',
|
| 64 |
+
aws_access_key_id=userdata.get('AWS_KEY'),
|
| 65 |
+
aws_secret_access_key=userdata.get('AWS_SECRET'))
|
| 66 |
for obj in s3.list_objects(Bucket='qarac')['Contents']:
|
| 67 |
filename = obj['Key']
|
| 68 |
s3.download_file('qarac',filename,'corpora/{}'.format(filename))
|
|
|
|
| 155 |
epoch_data[batch] = loss.item()
|
| 156 |
history[epoch_label] = epoch_data
|
| 157 |
scheduler.step()
|
| 158 |
+
huggingface_hub.login(token=userdata.get('HUGGINGFACE_TOKEN'))
|
| 159 |
trainer.question_encoder.push_to_hub('{}/qarac-roberta-question-encoder'.format(path))
|
| 160 |
trainer.answer_encoder.push_to_hub('{}/qarac-roberta-answer-encoder'.format(path))
|
| 161 |
trainer.decoder.push_to_hub('{}/qarac-roberta-decoder'.format(path))
|