File size: 5,334 Bytes
dffe123 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 | #version 450
#define KSTRIP_LEN 32
#define BLOCK_SIZE 64 // the output channel shoule be aligned to 64.
#define WARP 32
#define INNER_THREAD 16 // inner thread
#define ALL_THREAD 256
#define A_INSTRIP 8
#define A_STRIP 8 // (BLOCK_SIZE/A_INSTRIP)
#define B_INSTRIP 4 // (ALL_THREAD/BLOCK_SIZE)
#define B_STRIP 8 // (KSTRIP_LEN/B_INSTRIP)
#define PER_THREAD (BLOCK_SIZE/INNER_THREAD)
layout(binding = 0) readonly buffer Input0{
float image_data[];
};
layout(binding = 1) readonly buffer Input1 {
float bias_data[];
};
layout(binding = 2) readonly buffer Input2{
float weight_data[];
};
layout(binding = 3) writeonly buffer Output{
float output_data[];
};
layout(binding = 4) uniform pushBlock {
int Hi; // H in
int Wi; // W in
int H0; // H out
int W0; // W out
int stride_h;
int stride_w;
int pad_h;
int pad_w;
int Hk;
int Wk;
int dilation_h;
int dilation_w;
int Kg;
int Cg;
int group;
int CgHkWk_aligned;
int activationType; // 0 : no activation, 1: ReLU, 2: ReLU6.
int batchi; // batch index
int groupi; // group index
} p;
shared float wshare[KSTRIP_LEN][BLOCK_SIZE]; // 2 KB
shared float inshare[BLOCK_SIZE][KSTRIP_LEN]; // 2 KB
layout(local_size_x = ALL_THREAD, local_size_y = 1, local_size_z = 1) in;
void main()
{
int M = p.Kg; // output channel
int K = p.CgHkWk_aligned; // Hk * Wk * G // aligned to KSTRIP_LEN
int N = p.H0 * p.W0; // H0 * W0
int mIndex = int(gl_WorkGroupID.x) * BLOCK_SIZE;
int nIndex = int(gl_WorkGroupID.y) * BLOCK_SIZE;
int local_x = int(gl_LocalInvocationID.x) % 16; // 0~7
int local_y = int(gl_LocalInvocationID.x) / 16; // 0~31
int w_local_x = int(gl_LocalInvocationID.x) % KSTRIP_LEN; // 256 / 32 = 8
int w_local_y = int(gl_LocalInvocationID.x) / KSTRIP_LEN;
int in_local_x = int(gl_LocalInvocationID.x) % BLOCK_SIZE; // 256 / 64 = 4
int in_local_y = int(gl_LocalInvocationID.x) / BLOCK_SIZE;
int woffset = p.groupi * p.Kg * K + K * mIndex + w_local_y * K + w_local_x;
int inoffset = (p.batchi * p.group + p.groupi) * p.Hi*p.Wi*p.Cg;
int outoffset = (p.batchi * p.group + p.groupi) * p.H0*p.W0*p.Kg;
int biasoffset = p.groupi * p.Kg + mIndex + local_x * PER_THREAD;
vec4 sum[PER_THREAD];
{
for (int i = 0; i < PER_THREAD; i++)
{
sum[i] = vec4(bias_data[biasoffset + i]);
}
}
float regA[PER_THREAD];
float regB[PER_THREAD];
int KStrip = K / KSTRIP_LEN;
int KRemain = K - KStrip * KSTRIP_LEN; // NOTE: this value shoule be always 0.
for (int i = 0; i < KStrip; i++)
{
int k = i * KSTRIP_LEN;
// load Weight to local memory.
for (int s = 0; s < A_STRIP; s++)
{
wshare[w_local_x][s * A_INSTRIP + w_local_y] = weight_data[woffset + s * A_INSTRIP * K + k];
}
// load Input to local memory
for (int s = 0; s < B_STRIP; s++)
{
int kk = in_local_y + s * B_INSTRIP + k;
int HkWk = p.Hk * p.Wk;
int cg = kk/HkWk;
int hkwk = kk - cg * HkWk;
int hk = hkwk / p.Wk;
int wk = hkwk - hk * p.Wk;
int dh = hk * p.dilation_h, dw = wk * p.dilation_w;
int h0w0 = in_local_x + nIndex;
int h0 = h0w0 / p.W0;
int w0 = h0w0 - h0 * p.W0;
int hi = h0 * p.stride_h + dh - p.pad_h;
int wi = w0 * p.stride_w + dw - p.pad_w;
if (cg < p.Cg && uint(hi) < uint(p.Hi) && uint(wi) < uint(p.Wi))
inshare[in_local_x][s * B_INSTRIP + in_local_y] = image_data[inoffset + cg * p.Hi * p.Wi + hi * p.Wi + wi];
else
inshare[in_local_x][s * B_INSTRIP + in_local_y] = 0.f;
}
barrier();
for (int j = 0; j < KSTRIP_LEN; j++)
{
// Load shared memory to register.
for (int m = 0; m < 4; m++)
{
regA[m] = wshare[j][local_x*4 + m];
}
for (int m = 0; m < 4; m++)
{
regB[m] = inshare[local_y + 16 * m][j];
}
for (int m = 0; m < 4; m++)
{
for (int n = 0; n < 4; n++)
{
sum[m][n] += regA[m] * regB[n];
}
}
}
barrier();
}
if (p.activationType == 1) // ReLU
{
sum[0] = max(sum[0], vec4(0));
sum[1] = max(sum[1], vec4(0));
sum[2] = max(sum[2], vec4(0));
sum[3] = max(sum[3], vec4(0));
}
else if (p.activationType == 2) // ReLU6
{
sum[0] = clamp(sum[0], vec4(0), vec4(6));
sum[1] = clamp(sum[1], vec4(0), vec4(6));
sum[2] = clamp(sum[2], vec4(0), vec4(6));
sum[3] = clamp(sum[3], vec4(0), vec4(6));
}
for (int n = 0; n < PER_THREAD; n++)
{
int nIndex2 = nIndex + n * INNER_THREAD + local_y;
if (nIndex2 < N)
{
for (int m = 0; m < PER_THREAD; m++)
{
int mIndex2 = mIndex + local_x * PER_THREAD + m;
if (mIndex2 < M)
{
output_data[outoffset + mIndex2 * N + nIndex2] = sum[m][n];
}
}
}
}
}
|