Hugging Face's logo

qninhdt
/
mini-agent

Model card Files
xet
 Community
qninhdt commited on
Commit
8b94669
·
1 Parent(s): 2c9e8bc

cc

Browse files
Files changed (4) hide show
  1. configs/trainer/default.yaml +1 -1
  2. src/data/mixed_dataset.py +1 -1
  3. src/models/miniagent_module.py +22 -1
  4. test_bert.ipynb +130 -0
configs/trainer/default.yaml CHANGED
@@ -3,7 +3,7 @@ _target_: lightning.pytorch.trainer.Trainer
3
  default_root_dir: ${paths.output_dir}
4
 
5
  min_epochs: 1 # prevents early stopping
6
- max_epochs: 10
7
 
8
  accelerator: cpu
9
  devices: 1
 
3
  default_root_dir: ${paths.output_dir}
4
 
5
  min_epochs: 1 # prevents early stopping
6
+ max_epochs: 20
7
 
8
  accelerator: cpu
9
  devices: 1
src/data/mixed_dataset.py CHANGED
@@ -70,7 +70,7 @@ class MixedDataset(Dataset):
70
  correct_tool_mask = torch.tensor(
71
  [1] * len(correct_tools)
72
  + [0] * (self.tool_capacity - len(correct_tools)),
73
- dtype=torch.long,
74
  )
75
 
76
  tools = correct_tools + wrong_tools
 
70
  correct_tool_mask = torch.tensor(
71
  [1] * len(correct_tools)
72
  + [0] * (self.tool_capacity - len(correct_tools)),
73
+ dtype=torch.bool,
74
  )
75
 
76
  tools = correct_tools + wrong_tools
src/models/miniagent_module.py CHANGED
@@ -34,6 +34,8 @@ class MiniAgentModule(LightningModule):
34
  self.pred_model = pred_model
35
 
36
  self.val_acc = Accuracy(task="binary")
 
 
37
 
38
  self.lr = lr
39
 
@@ -106,12 +108,31 @@ class MiniAgentModule(LightningModule):
106
  pred = pred.view(B, C)
107
  pred = torch.sigmoid(pred)
108
 
109
- pred_tool_mask = (pred > 0.5).long()
110
  pos_sample = (pred_tool_mask == correct_tool_mask).all(dim=1).long()
111
 
 
 
 
 
 
 
 
112
  self.val_acc.update(pos_sample, torch.ones_like(pos_sample))
 
 
113
 
114
  self.log("val/acc", self.val_acc, on_epoch=True, sync_dist=True, prog_bar=True)
 
 
 
 
 
 
 
 
 
 
115
 
116
  def on_validation_epoch_end(self) -> None:
117
  pass
 
34
  self.pred_model = pred_model
35
 
36
  self.val_acc = Accuracy(task="binary")
37
+ self.val_precision = MeanMetric()
38
+ self.val_recall = MeanMetric()
39
 
40
  self.lr = lr
41
 
 
108
  pred = pred.view(B, C)
109
  pred = torch.sigmoid(pred)
110
 
111
+ pred_tool_mask = pred > 0.5
112
  pos_sample = (pred_tool_mask == correct_tool_mask).all(dim=1).long()
113
 
114
+ true_pos_mask = pred_tool_mask & correct_tool_mask
115
+
116
+ precision = true_pos_mask.sum(dim=1) / torch.clamp(
117
+ pred_tool_mask.sum(dim=1), min=1
118
+ )
119
+ recall = true_pos_mask.sum(dim=1) / correct_tool_mask.sum(dim=1)
120
+
121
  self.val_acc.update(pos_sample, torch.ones_like(pos_sample))
122
+ self.val_precision.update(precision)
123
+ self.val_recall.update(recall)
124
 
125
  self.log("val/acc", self.val_acc, on_epoch=True, sync_dist=True, prog_bar=True)
126
+ self.log(
127
+ "val/precision",
128
+ self.val_precision,
129
+ on_epoch=True,
130
+ sync_dist=True,
131
+ prog_bar=True,
132
+ )
133
+ self.log(
134
+ "val/recall", self.val_recall, on_epoch=True, sync_dist=True, prog_bar=True
135
+ )
136
 
137
  def on_validation_epoch_end(self) -> None:
138
  pass
test_bert.ipynb CHANGED
@@ -1,5 +1,135 @@
1
  {
2
  "cells": [
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3
  {
4
  "cell_type": "code",
5
  "execution_count": 1,
 
1
  {
2
  "cells": [
3
+ {
4
+ "cell_type": "code",
5
+ "execution_count": 4,
6
+ "metadata": {},
7
+ "outputs": [
8
+ {
9
+ "data": {
10
+ "text/plain": [
11
+ "BaseModelOutputWithPoolingAndCrossAttentions(last_hidden_state=tensor([[[ 0.1386, 0.1583, -0.2967, ..., -0.2709, -0.2844, 0.4581],\n",
12
+ " [ 0.5364, -0.2327, 0.1754, ..., 0.5540, 0.4981, -0.0024],\n",
13
+ " [ 0.3002, -0.3475, 0.1208, ..., -0.4562, 0.3288, 0.8773],\n",
14
+ " ...,\n",
15
+ " [ 0.3799, 0.1203, 0.8283, ..., -0.8624, -0.5957, 0.0471],\n",
16
+ " [-0.0252, -0.7177, -0.6950, ..., 0.0757, -0.6668, -0.3401],\n",
17
+ " [ 0.7535, 0.2391, 0.0717, ..., 0.2467, -0.6458, -0.3213]]],\n",
18
+ " grad_fn=<NativeLayerNormBackward0>), pooler_output=tensor([[-0.9377, -0.5043, -0.9799, 0.9030, 0.9329, -0.2438, 0.8926, 0.2288,\n",
19
+ " -0.9531, -1.0000, -0.8862, 0.9906, 0.9855, 0.7155, 0.9455, -0.8645,\n",
20
+ " -0.6035, -0.6666, 0.3020, -0.1587, 0.7455, 1.0000, -0.4022, 0.4261,\n",
21
+ " 0.6151, 0.9996, -0.8773, 0.9594, 0.9585, 0.6950, -0.6718, 0.3325,\n",
22
+ " -0.9954, -0.2268, -0.9658, -0.9951, 0.6127, -0.7670, 0.0873, 0.0824,\n",
23
+ " -0.9518, 0.4713, 1.0000, 0.3299, 0.7583, -0.2670, -1.0000, 0.3166,\n",
24
+ " -0.9364, 0.9910, 0.9719, 0.9893, 0.2190, 0.6048, 0.5849, -0.4123,\n",
25
+ " -0.0063, 0.1719, -0.3988, -0.6190, -0.6603, 0.5069, -0.9757, -0.9039,\n",
26
+ " 0.9926, 0.9323, -0.3687, -0.4869, -0.3143, 0.0499, 0.9129, 0.3396,\n",
27
+ " -0.1879, -0.9235, 0.8675, 0.3228, -0.6406, 1.0000, -0.7989, -0.9931,\n",
28
+ " 0.9629, 0.9124, 0.4827, -0.7276, 0.5996, -1.0000, 0.7548, -0.1600,\n",
29
+ " -0.9941, 0.3386, 0.8394, -0.4158, 0.2943, 0.6111, -0.5745, -0.7185,\n",
30
+ " -0.4768, -0.9681, -0.4327, -0.6732, 0.1248, -0.2093, -0.5882, -0.4186,\n",
31
+ " 0.5447, -0.6125, -0.6138, 0.4712, 0.4779, 0.7633, 0.3974, -0.4148,\n",
32
+ " 0.7063, -0.9680, 0.7389, -0.4270, -0.9948, -0.6019, -0.9950, 0.7459,\n",
33
+ " -0.6343, -0.2753, 0.9522, -0.5724, 0.6218, -0.1295, -0.9905, -1.0000,\n",
34
+ " -0.8710, -0.7506, -0.5008, -0.4827, -0.9872, -0.9847, 0.7214, 0.9694,\n",
35
+ " 0.3013, 1.0000, -0.4427, 0.9699, -0.5431, -0.8189, 0.9180, -0.5132,\n",
36
+ " 0.9026, 0.5274, -0.5940, 0.2928, -0.6933, 0.7179, -0.9318, -0.2776,\n",
37
+ " -0.9160, -0.9457, -0.3287, 0.9556, -0.7927, -0.9860, -0.1904, -0.2760,\n",
38
+ " -0.6062, 0.9005, 0.9266, 0.4353, -0.6858, 0.4720, 0.2851, 0.7685,\n",
39
+ " -0.8647, -0.5626, 0.5127, -0.5468, -0.9490, -0.9907, -0.5809, 0.7146,\n",
40
+ " 0.9948, 0.7981, 0.3463, 0.9349, -0.4238, 0.9333, -0.9754, 0.9936,\n",
41
+ " -0.2597, 0.4665, -0.5400, 0.4947, -0.8723, 0.0034, 0.8378, -0.9134,\n",
42
+ " -0.8432, -0.2516, -0.5177, -0.4687, -0.9491, 0.5691, -0.4856, -0.4857,\n",
43
+ " -0.2245, 0.9609, 0.9823, 0.7496, 0.6256, 0.8552, -0.9073, -0.5802,\n",
44
+ " 0.2874, 0.3017, 0.3016, 0.9974, -0.8503, -0.2108, -0.9261, -0.9907,\n",
45
+ " -0.0252, -0.9488, -0.3972, -0.8097, 0.8707, -0.7512, 0.8107, 0.5488,\n",
46
+ " -0.9830, -0.8569, 0.4852, -0.6156, 0.4846, -0.2893, 0.9647, 0.9858,\n",
47
+ " -0.7064, 0.7120, 0.9593, -0.9590, -0.8708, 0.7893, -0.3561, 0.8603,\n",
48
+ " -0.7243, 0.9882, 0.9876, 0.9282, -0.9547, -0.8329, -0.7993, -0.8398,\n",
49
+ " -0.2333, 0.2315, 0.9712, 0.6055, 0.6388, 0.2429, -0.7884, 0.9981,\n",
50
+ " -0.9448, -0.9804, -0.8184, -0.3534, -0.9951, 0.9729, 0.4165, 0.8094,\n",
51
+ " -0.6227, -0.8183, -0.9817, 0.8532, 0.1242, 0.9826, -0.6376, -0.9450,\n",
52
+ " -0.8094, -0.9748, 0.0412, -0.3097, -0.8153, -0.0306, -0.9255, 0.5677,\n",
53
+ " 0.6217, 0.6652, -0.9682, 0.9997, 1.0000, 0.9826, 0.9013, 0.8950,\n",
54
+ " -1.0000, -0.8081, 1.0000, -0.9995, -1.0000, -0.9361, -0.8200, 0.4755,\n",
55
+ " -1.0000, -0.2698, -0.0111, -0.9297, 0.8492, 0.9879, 0.9950, -1.0000,\n",
56
+ " 0.8653, 0.9513, -0.5679, 0.9966, -0.6713, 0.9815, 0.6008, 0.7414,\n",
57
+ " -0.3265, 0.5574, -0.9801, -0.8956, -0.8082, -0.9267, 0.9999, 0.2542,\n",
58
+ " -0.7970, -0.8854, 0.7831, -0.1391, -0.0060, -0.9786, -0.4503, 0.8895,\n",
59
+ " 0.9021, 0.3021, 0.2650, -0.5750, 0.5099, 0.1216, 0.1170, 0.6484,\n",
60
+ " -0.9505, -0.3889, -0.6938, 0.2508, -0.7526, -0.9831, 0.9646, -0.2742,\n",
61
+ " 0.9865, 1.0000, 0.3756, -0.9045, 0.8847, 0.4860, -0.5515, 1.0000,\n",
62
+ " 0.9092, -0.9904, -0.4959, 0.7900, -0.7156, -0.8280, 0.9999, -0.4197,\n",
63
+ " -0.9282, -0.7733, 0.9945, -0.9956, 0.9998, -0.8985, -0.9838, 0.9735,\n",
64
+ " 0.9655, -0.8103, -0.8325, 0.1020, -0.6722, 0.4561, -0.9412, 0.8396,\n",
65
+ " 0.6979, -0.1201, 0.9288, -0.8345, -0.6312, 0.4356, -0.8901, -0.4565,\n",
66
+ " 0.9874, 0.5709, -0.2111, -0.0206, -0.4182, -0.9116, -0.9781, 0.8246,\n",
67
+ " 1.0000, -0.4229, 0.9489, -0.5226, -0.0986, 0.2202, 0.7459, 0.7152,\n",
68
+ " -0.3528, -0.8800, 0.9299, -0.9716, -0.9949, 0.7278, 0.2206, -0.4944,\n",
69
+ " 1.0000, 0.6285, 0.3795, 0.7228, 0.9993, 0.0301, 0.5936, 0.9816,\n",
70
+ " 0.9914, -0.3465, 0.5882, 0.8365, -0.9824, -0.4488, -0.7612, 0.1331,\n",
71
+ " -0.9479, -0.0559, -0.9697, 0.9846, 0.9960, 0.5818, 0.3121, 0.8577,\n",
72
+ " 1.0000, -0.9274, 0.6693, -0.1365, 0.8035, -1.0000, -0.8057, -0.4504,\n",
73
+ " -0.1711, -0.9512, -0.5899, 0.3991, -0.9754, 0.9563, 0.8806, -0.9937,\n",
74
+ " -0.9923, -0.4979, 0.8853, 0.1439, -0.9994, -0.8986, -0.6272, 0.8385,\n",
75
+ " -0.3239, -0.9470, -0.7009, -0.4768, 0.5742, -0.2216, 0.5665, 0.9667,\n",
76
+ " 0.7935, -0.9401, -0.6746, -0.1753, -0.9163, 0.9409, -0.8701, -0.9894,\n",
77
+ " -0.2514, 1.0000, -0.4087, 0.9385, 0.6050, 0.8219, -0.2712, 0.3326,\n",
78
+ " 0.9827, 0.3613, -0.8314, -0.9850, -0.2861, -0.5398, 0.8254, 0.8414,\n",
79
+ " 0.7590, 0.9412, 0.9627, 0.2765, -0.0737, 0.0399, 0.9998, -0.3095,\n",
80
+ " -0.1933, -0.4689, -0.2511, -0.4629, -0.2914, 1.0000, 0.3963, 0.7777,\n",
81
+ " -0.9950, -0.9808, -0.9303, 1.0000, 0.8822, -0.6848, 0.8124, 0.6242,\n",
82
+ " -0.2551, 0.8266, -0.2791, -0.3167, 0.2294, 0.1682, 0.9627, -0.6738,\n",
83
+ " -0.9904, -0.7910, 0.7099, -0.9770, 1.0000, -0.7030, -0.3960, -0.5981,\n",
84
+ " -0.6683, -0.2727, -0.0183, -0.9882, -0.3841, 0.5605, 0.9745, 0.3505,\n",
85
+ " -0.4898, -0.9298, 0.9578, 0.9533, -0.9859, -0.9597, 0.9777, -0.9784,\n",
86
+ " 0.7550, 1.0000, 0.3446, 0.6786, 0.3947, -0.5349, 0.5541, -0.6754,\n",
87
+ " 0.8078, -0.9595, -0.4484, -0.3901, 0.3983, -0.1319, -0.2896, 0.7860,\n",
88
+ " 0.3500, -0.5530, -0.7294, -0.2361, 0.4663, 0.9332, -0.3048, -0.1916,\n",
89
+ " 0.2318, -0.3230, -0.9323, -0.4672, -0.6315, -1.0000, 0.8068, -1.0000,\n",
90
+ " 0.8035, 0.4066, -0.3700, 0.8760, 0.7829, 0.8298, -0.8628, -0.9795,\n",
91
+ " 0.1322, 0.8529, -0.5029, -0.9057, -0.6918, 0.5017, -0.2052, 0.1564,\n",
92
+ " -0.7397, 0.8156, -0.3414, 1.0000, 0.2659, -0.8292, -0.9821, 0.2491,\n",
93
+ " -0.3009, 1.0000, -0.8952, -0.9832, 0.3330, -0.9180, -0.8493, 0.5868,\n",
94
+ " 0.1653, -0.8522, -0.9961, 0.9220, 0.8661, -0.6477, 0.7927, -0.3991,\n",
95
+ " -0.7691, 0.1512, 0.9868, 0.9924, 0.7317, 0.9083, -0.1226, -0.5258,\n",
96
+ " 0.9840, 0.4009, -0.0436, 0.1361, 1.0000, 0.4004, -0.9497, -0.1309,\n",
97
+ " -0.9788, -0.3522, -0.9551, 0.3755, 0.3099, 0.9195, -0.4460, 0.9738,\n",
98
+ " -0.9714, 0.1901, -0.8894, -0.7863, 0.4757, -0.9463, -0.9892, -0.9938,\n",
99
+ " 0.8142, -0.4077, -0.1895, 0.2102, 0.1715, 0.6322, 0.5566, -1.0000,\n",
100
+ " 0.9642, 0.6150, 0.9768, 0.9768, 0.9115, 0.8108, 0.3251, -0.9920,\n",
101
+ " -0.9910, -0.5438, -0.3567, 0.7960, 0.7648, 0.8900, 0.6470, -0.4875,\n",
102
+ " -0.4792, -0.7756, -0.8423, -0.9972, 0.5961, -0.8679, -0.9678, 0.9718,\n",
103
+ " -0.3461, -0.1534, -0.2139, -0.9586, 0.9321, 0.7627, 0.4636, 0.0862,\n",
104
+ " 0.5071, 0.9170, 0.9597, 0.9882, -0.9231, 0.8555, -0.9196, 0.6712,\n",
105
+ " 0.9381, -0.9606, 0.2335, 0.8301, -0.5560, 0.3696, -0.4752, -0.9740,\n",
106
+ " 0.8174, -0.4268, 0.7773, -0.4798, 0.0639, -0.4718, -0.2607, -0.7624,\n",
107
+ " -0.8742, 0.6576, 0.6207, 0.9219, 0.9360, -0.0496, -0.8942, -0.3701,\n",
108
+ " -0.8944, -0.9526, 0.9536, -0.0851, -0.2961, 0.9031, 0.1321, 0.9324,\n",
109
+ " 0.4289, -0.4989, -0.4174, -0.7639, 0.8887, -0.7894, -0.7639, -0.7093,\n",
110
+ " 0.8105, 0.3595, 1.0000, -0.9188, -0.9878, -0.8268, -0.6012, 0.4992,\n",
111
+ " -0.7880, -1.0000, 0.3609, -0.8314, 0.8524, -0.9398, 0.9500, -0.9339,\n",
112
+ " -0.9851, -0.3495, 0.8436, 0.9375, -0.5159, -0.8989, 0.5196, -0.8797,\n",
113
+ " 0.9979, 0.8753, -0.8277, -0.0012, 0.6013, -0.9184, -0.7398, 0.9228]],\n",
114
+ " grad_fn=<TanhBackward0>), hidden_states=None, past_key_values=None, attentions=None, cross_attentions=None)"
115
+ ]
116
+ },
117
+ "execution_count": 4,
118
+ "metadata": {},
119
+ "output_type": "execute_result"
120
+ }
121
+ ],
122
+ "source": [
123
+ "from transformers import BertTokenizer, BertModel\n",
124
+ "\n",
125
+ "tokenizer = BertTokenizer.from_pretrained(\"bert-base-uncased\")\n",
126
+ "model = BertModel.from_pretrained(\"bert-base-uncased\")\n",
127
+ "text = \"Replace me by any text you'd like.\"\n",
128
+ "encoded_input = tokenizer(text, return_tensors=\"pt\")\n",
129
+ "output = model(**encoded_input)\n",
130
+ "output"
131
+ ]
132
+ },
133
  {
134
  "cell_type": "code",
135
  "execution_count": 1,