Hugging Face's logo

Spaces:
daniel-de-leon
/
test-suite
Running

App Files Community
daniel-de-leon commited on
Commit
06973e4
·
1 Parent(s): 6a9cefc

just test one subtask

Browse files
Files changed (1) hide show
  1. glue-suite-v2.py +5 -115
glue-suite-v2.py CHANGED
@@ -10,127 +10,17 @@ class Suite(evaluate.EvaluationSuite):
10
  self.suite = [
11
  SubTask(
12
  task_type="text-classification",
13
- data="glue",
14
- subset="cola",
15
  split="test[:10]",
16
  args_for_task={
17
  "metric": "accuracy",
18
  "input_column": "sentence",
19
  "label_column": "label",
20
  "label_mapping": {
21
- "LABEL_0": 0.0,
22
- "LABEL_1": 1.0
23
- }
24
- }
25
- ),
26
- SubTask(
27
- task_type="text-classification",
28
- data="glue",
29
- subset="sst2",
30
- split="validation[:10]",
31
- args_for_task={
32
- "metric": "accuracy",
33
- "input_column": "sentence",
34
- "label_column": "label",
35
- "label_mapping": {
36
- "LABEL_0": 0.0,
37
- "LABEL_1": 1.0
38
- }
39
- }
40
- ),
41
- SubTask(
42
- task_type="text-classification",
43
- data="glue",
44
- subset="qqp",
45
- split="validation[:10]",
46
- args_for_task={
47
- "metric": "accuracy",
48
- "input_column": "question1",
49
- "second_input_column": "question2",
50
- "label_column": "label",
51
- "label_mapping": {
52
- "LABEL_0": 0,
53
- "LABEL_1": 1
54
- }
55
- }
56
- ),
57
- SubTask(
58
- task_type="text-classification",
59
- data="glue",
60
- subset="mrpc",
61
- split="validation[:10]",
62
- args_for_task={
63
- "metric": "accuracy",
64
- "input_column": "sentence1",
65
- "second_input_column": "sentence2",
66
- "label_column": "label",
67
- "label_mapping": {
68
- "LABEL_0": 0,
69
- "LABEL_1": 1
70
- }
71
- }
72
- ),
73
- SubTask(
74
- task_type="text-classification",
75
- data="glue",
76
- subset="mnli",
77
- split="validation_mismatched[:10]",
78
- args_for_task={
79
- "metric": "accuracy",
80
- "input_column": "premise",
81
- "second_input_column": "hypothesis",
82
- "label_mapping": {
83
- "LABEL_0": 0,
84
- "LABEL_1": 1,
85
- "LABEL_2": 2
86
- }
87
- }
88
- ),
89
- SubTask(
90
- task_type="text-classification",
91
- data="glue",
92
- subset="qnli",
93
- split="validation[:10]",
94
- args_for_task={
95
- "metric": "accuracy",
96
- "input_column": "question",
97
- "second_input_column": "sentence",
98
- "label_column": "label",
99
- "label_mapping": {
100
- "LABEL_0": 0,
101
- "LABEL_1": 1
102
- }
103
- }
104
- ),
105
- SubTask(
106
- task_type="text-classification",
107
- data="glue",
108
- subset="rte",
109
- split="validation[:10]",
110
- args_for_task={
111
- "metric": "accuracy",
112
- "input_column": "sentence1",
113
- "second_input_column": "sentence2",
114
- "label_column": "label",
115
- "label_mapping": {
116
- "LABEL_0": 0,
117
- "LABEL_1": 1
118
- }
119
- }
120
- ),
121
- SubTask(
122
- task_type="text-classification",
123
- data="glue",
124
- subset="wnli",
125
- split="validation[:10]",
126
- args_for_task={
127
- "metric": "accuracy",
128
- "input_column": "sentence1",
129
- "second_input_column": "sentence2",
130
- "label_column": "label",
131
- "label_mapping": {
132
- "LABEL_0": 0,
133
- "LABEL_1": 1
134
  }
135
  }
136
  )
 
10
  self.suite = [
11
  SubTask(
12
  task_type="text-classification",
13
+ data="hate_speech",
 
14
  split="test[:10]",
15
  args_for_task={
16
  "metric": "accuracy",
17
  "input_column": "sentence",
18
  "label_column": "label",
19
  "label_mapping": {
20
+ "noHate": 0.0,
21
+ "hate": 1.0,
22
+ "relation": 1.0,
23
+ "idk/skip": 1.0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
24
  }
25
  }
26
  )