test / modules /dnn /src /vkcom /shader /conv_implicit_gemm.comp
tgollogly's picture
Upload 7492 files
dffe123 verified
Raw History Blame Contribute Delete
5.33 kB
#version 450
#define KSTRIP_LEN 32
#define BLOCK_SIZE 64 // the output channel shoule be aligned to 64.
#define WARP 32
#define INNER_THREAD 16 // inner thread
#define ALL_THREAD 256
#define A_INSTRIP 8
#define A_STRIP 8 // (BLOCK_SIZE/A_INSTRIP)
#define B_INSTRIP 4 // (ALL_THREAD/BLOCK_SIZE)
#define B_STRIP 8 // (KSTRIP_LEN/B_INSTRIP)
#define PER_THREAD (BLOCK_SIZE/INNER_THREAD)
layout(binding = 0) readonly buffer Input0{
float image_data[];
};
layout(binding = 1) readonly buffer Input1 {
float bias_data[];
};
layout(binding = 2) readonly buffer Input2{
float weight_data[];
};
layout(binding = 3) writeonly buffer Output{
float output_data[];
};
layout(binding = 4) uniform pushBlock {
int Hi; // H in
int Wi; // W in
int H0; // H out
int W0; // W out
int stride_h;
int stride_w;
int pad_h;
int pad_w;
int Hk;
int Wk;
int dilation_h;
int dilation_w;
int Kg;
int Cg;
int group;
int CgHkWk_aligned;
int activationType; // 0 : no activation, 1: ReLU, 2: ReLU6.
int batchi; // batch index
int groupi; // group index
} p;
shared float wshare[KSTRIP_LEN][BLOCK_SIZE]; // 2 KB
shared float inshare[BLOCK_SIZE][KSTRIP_LEN]; // 2 KB
layout(local_size_x = ALL_THREAD, local_size_y = 1, local_size_z = 1) in;
void main()
{
int M = p.Kg; // output channel
int K = p.CgHkWk_aligned; // Hk * Wk * G // aligned to KSTRIP_LEN
int N = p.H0 * p.W0; // H0 * W0
int mIndex = int(gl_WorkGroupID.x) * BLOCK_SIZE;
int nIndex = int(gl_WorkGroupID.y) * BLOCK_SIZE;
int local_x = int(gl_LocalInvocationID.x) % 16; // 0~7
int local_y = int(gl_LocalInvocationID.x) / 16; // 0~31
int w_local_x = int(gl_LocalInvocationID.x) % KSTRIP_LEN; // 256 / 32 = 8
int w_local_y = int(gl_LocalInvocationID.x) / KSTRIP_LEN;
int in_local_x = int(gl_LocalInvocationID.x) % BLOCK_SIZE; // 256 / 64 = 4
int in_local_y = int(gl_LocalInvocationID.x) / BLOCK_SIZE;
int woffset = p.groupi * p.Kg * K + K * mIndex + w_local_y * K + w_local_x;
int inoffset = (p.batchi * p.group + p.groupi) * p.Hi*p.Wi*p.Cg;
int outoffset = (p.batchi * p.group + p.groupi) * p.H0*p.W0*p.Kg;
int biasoffset = p.groupi * p.Kg + mIndex + local_x * PER_THREAD;
vec4 sum[PER_THREAD];
{
for (int i = 0; i < PER_THREAD; i++)
{
sum[i] = vec4(bias_data[biasoffset + i]);
}
}
float regA[PER_THREAD];
float regB[PER_THREAD];
int KStrip = K / KSTRIP_LEN;
int KRemain = K - KStrip * KSTRIP_LEN; // NOTE: this value shoule be always 0.
for (int i = 0; i < KStrip; i++)
{
int k = i * KSTRIP_LEN;
// load Weight to local memory.
for (int s = 0; s < A_STRIP; s++)
{
wshare[w_local_x][s * A_INSTRIP + w_local_y] = weight_data[woffset + s * A_INSTRIP * K + k];
}
// load Input to local memory
for (int s = 0; s < B_STRIP; s++)
{
int kk = in_local_y + s * B_INSTRIP + k;
int HkWk = p.Hk * p.Wk;
int cg = kk/HkWk;
int hkwk = kk - cg * HkWk;
int hk = hkwk / p.Wk;
int wk = hkwk - hk * p.Wk;
int dh = hk * p.dilation_h, dw = wk * p.dilation_w;
int h0w0 = in_local_x + nIndex;
int h0 = h0w0 / p.W0;
int w0 = h0w0 - h0 * p.W0;
int hi = h0 * p.stride_h + dh - p.pad_h;
int wi = w0 * p.stride_w + dw - p.pad_w;
if (cg < p.Cg && uint(hi) < uint(p.Hi) && uint(wi) < uint(p.Wi))
inshare[in_local_x][s * B_INSTRIP + in_local_y] = image_data[inoffset + cg * p.Hi * p.Wi + hi * p.Wi + wi];
else
inshare[in_local_x][s * B_INSTRIP + in_local_y] = 0.f;
}
barrier();
for (int j = 0; j < KSTRIP_LEN; j++)
{
// Load shared memory to register.
for (int m = 0; m < 4; m++)
{
regA[m] = wshare[j][local_x*4 + m];
}
for (int m = 0; m < 4; m++)
{
regB[m] = inshare[local_y + 16 * m][j];
}
for (int m = 0; m < 4; m++)
{
for (int n = 0; n < 4; n++)
{
sum[m][n] += regA[m] * regB[n];
}
}
}
barrier();
}
if (p.activationType == 1) // ReLU
{
sum[0] = max(sum[0], vec4(0));
sum[1] = max(sum[1], vec4(0));
sum[2] = max(sum[2], vec4(0));
sum[3] = max(sum[3], vec4(0));
}
else if (p.activationType == 2) // ReLU6
{
sum[0] = clamp(sum[0], vec4(0), vec4(6));
sum[1] = clamp(sum[1], vec4(0), vec4(6));
sum[2] = clamp(sum[2], vec4(0), vec4(6));
sum[3] = clamp(sum[3], vec4(0), vec4(6));
}
for (int n = 0; n < PER_THREAD; n++)
{
int nIndex2 = nIndex + n * INNER_THREAD + local_y;
if (nIndex2 < N)
{
for (int m = 0; m < PER_THREAD; m++)
{
int mIndex2 = mIndex + local_x * PER_THREAD + m;
if (mIndex2 < M)
{
output_data[outoffset + mIndex2 * N + nIndex2] = sum[m][n];
}
}
}
}
}