"
],
"text/html": [
"\n",
" \n",
" \n",
"
\n",
" [2988/2988 1:03:44, Epoch 2/2]\n",
"
\n",
" \n",
" \n",
" | Step | \n",
" Training Loss | \n",
" Validation Loss | \n",
" Accuracy | \n",
" Precision | \n",
" Recall | \n",
" F1 | \n",
"
\n",
" \n",
" \n",
" \n",
" | 1000 | \n",
" 0.061500 | \n",
" 0.056078 | \n",
" 0.980891 | \n",
" 0.963388 | \n",
" 0.986134 | \n",
" 0.974628 | \n",
"
\n",
" \n",
" | 2000 | \n",
" 0.046200 | \n",
" 0.044987 | \n",
" 0.987958 | \n",
" 0.982677 | \n",
" 0.985009 | \n",
" 0.983842 | \n",
"
\n",
" \n",
"
"
]
},
"metadata": {}
},
{
"output_type": "execute_result",
"data": {
"text/plain": [
"TrainOutput(global_step=2988, training_loss=0.04876313854253276, metrics={'train_runtime': 3825.3887, 'train_samples_per_second': 24.988, 'train_steps_per_second': 0.781, 'total_flos': 1.1653032603057624e+16, 'train_loss': 0.04876313854253276, 'epoch': 2.0})"
]
},
"metadata": {},
"execution_count": 64
}
],
"execution_count": 64
},
{
"cell_type": "markdown",
"source": [
"### 5. Evaluation\n"
],
"metadata": {
"id": "QStfued8ohLi"
}
},
{
"cell_type": "markdown",
"source": [
"run the compute_metrics on the test dataset."
],
"metadata": {
"id": "CyShT0moeK6V"
}
},
{
"cell_type": "code",
"source": [
"datasets[\"test\"]"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "s4erSLYTjjcv",
"outputId": "a4d8333d-008b-4185-9aa5-60e1f4fd7f68"
},
"execution_count": 100,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"Dataset({\n",
" features: ['text', 'labels', 'text_length', 'ttr', 'punctuation_count', 'stopword_ratio', '__index_level_0__', 'input_ids', 'attention_mask'],\n",
" num_rows: 20000\n",
"})"
]
},
"metadata": {},
"execution_count": 100
}
]
},
{
"cell_type": "code",
"source": [],
"metadata": {
"id": "qnbmoqVtofi6"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"test_metrics = trainer.evaluate(\n",
" eval_dataset=datasets[\"test\"],\n",
" metric_key_prefix=\"test\"\n",
")\n",
"\n",
"print(\"Test set metrics:\")\n",
"for key, value in test_metrics.items():\n",
" print(f\" {key}: {value:.4f}\")\n"
],
"metadata": {
"trusted": true,
"colab": {
"base_uri": "https://localhost:8080/",
"height": 211
},
"id": "u-Tv4aUN-J3k",
"outputId": "05a737e6-f0de-46e8-cd9e-a87af574ef2a"
},
"outputs": [
{
"output_type": "display_data",
"data": {
"text/plain": [
""
],
"text/html": [
"\n",
" \n",
" \n",
"
\n",
" [ 197/1597 30:56 < 3:41:02, 0.11 it/s]\n",
"