Hugging Face's logo

theapemachine
/
sparse-transformer-experiments

Model card Files
xet
 Community
sparse-transformer-experiments
File size: 2,350 Bytes
fe3111e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
{
  "sparse_ema": {
    "ml": 5.8750211715698235,
    "sl": 0.24326497850071485,
    "rs": [
      {
        "vl": 5.703006855646769,
        "vp": 299.76740451157195,
        "wall": 181.73005015598028,
        "ms": 363.46010031196056,
        "np": 153625681,
        "tl": 5.23537540435791,
        "ov": []
      },
      {
        "vl": 6.047035487492879,
        "vp": 422.85760344611595,
        "wall": 181.41676819100394,
        "ms": 362.8335363820079,
        "np": 153625681,
        "tl": 5.769244194030762,
        "ov": []
      }
    ],
    "ms": 363.1468183469842
  },
  "dense_same": {
    "ml": 5.470963843663534,
    "sl": 0.011881365182615072,
    "rs": [
      {
        "vl": 5.462562449773153,
        "vp": 235.7006222090039,
        "wall": 182.27416944300057,
        "ms": 364.54833888600115,
        "np": 153625681,
        "tl": 4.8674397468566895,
        "ov": []
      },
      {
        "vl": 5.479365237553915,
        "vp": 239.69451000129558,
        "wall": 182.27069638599642,
        "ms": 364.54139277199283,
        "np": 153625681,
        "tl": 4.805025577545166,
        "ov": []
      }
    ],
    "ms": 364.544865828997
  },
  "dense_350steps": {
    "ml": 5.671385749181112,
    "sl": 0.00022165870568799145,
    "rs": [
      {
        "vl": 5.671229012807211,
        "vp": 290.39120968339057,
        "wall": 127.52659200300695,
        "ms": 364.36169143716273,
        "np": 153625681,
        "tl": 4.9916605949401855,
        "ov": []
      },
      {
        "vl": 5.671542485555013,
        "vp": 290.4822536829884,
        "wall": 127.42596015898744,
        "ms": 364.07417188282125,
        "np": 153625681,
        "tl": 4.939430236816406,
        "ov": []
      }
    ],
    "ms": 364.217931659992
  },
  "dense_ffn1": {
    "ml": 5.632929547627767,
    "sl": 0.012743296332561443,
    "rs": [
      {
        "vl": 5.641940418879191,
        "vp": 282.0094042761895,
        "wall": 142.137044994015,
        "ms": 284.27408998803,
        "np": 128447569,
        "tl": 5.103222370147705,
        "ov": []
      },
      {
        "vl": 5.623918676376343,
        "vp": 276.97262549508514,
        "wall": 142.10350732298684,
        "ms": 284.2070146459737,
        "np": 128447569,
        "tl": 4.966079235076904,
        "ov": []
      }
    ],
    "ms": 284.24055231700186
  }
}