Hugging Face's logo

tda45
/
TdAI

Safetensors
GGUF
Turkish
llama
Llama-3
instruct
finetune
chatml
gpt4
synthetic data
distillation
function calling
json mode
axolotl
roleplaying
chat
Model card Files
xet
 Community
tda45 commited on
Commit
c9c3262
·
verified ·
1 Parent(s): 20a07ae

Upload folder using huggingface_hub (part 5)

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32.cl +164 -0
  2. llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32_flat.cl +202 -0
  3. llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32.cl +144 -0
  4. llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32_flat.cl +167 -0
  5. llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32.cl +141 -0
  6. llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32_flat.cl +190 -0
  7. llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32.cl +192 -0
  8. llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_16x_flat.cl +307 -0
  9. llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_8x_flat.cl +265 -0
  10. llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_8x_flat.cl +272 -0
  11. llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_v.cl +254 -0
  12. llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_q4_1_f32.cl +219 -0
  13. llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_q4_1_f32_flat.cl +229 -0
  14. llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32.cl +180 -0
  15. llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32_flat.cl +196 -0
  16. llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32.cl +241 -0
  17. llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32_flat.cl +243 -0
  18. llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32.cl +243 -0
  19. llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32_flat.cl +247 -0
  20. llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32.cl +187 -0
  21. llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32_flat.cl +203 -0
  22. llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32.cl +194 -0
  23. llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32_flat.cl +178 -0
  24. llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_q8_0_f32.cl +125 -0
  25. llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_q8_0_f32_flat.cl +202 -0
  26. llama.cpp/ggml/src/ggml-opencl/kernels/neg.cl +125 -0
  27. llama.cpp/ggml/src/ggml-opencl/kernels/norm.cl +164 -0
  28. llama.cpp/ggml/src/ggml-opencl/kernels/pad.cl +39 -0
  29. llama.cpp/ggml/src/ggml-opencl/kernels/relu.cl +16 -0
  30. llama.cpp/ggml/src/ggml-opencl/kernels/repeat.cl +38 -0
  31. llama.cpp/ggml/src/ggml-opencl/kernels/rms_norm.cl +190 -0
  32. llama.cpp/ggml/src/ggml-opencl/kernels/rope.cl +747 -0
  33. llama.cpp/ggml/src/ggml-opencl/kernels/scale.cl +27 -0
  34. llama.cpp/ggml/src/ggml-opencl/kernels/set_rows.cl +708 -0
  35. llama.cpp/ggml/src/ggml-opencl/kernels/sigmoid.cl +29 -0
  36. llama.cpp/ggml/src/ggml-opencl/kernels/silu.cl +30 -0
  37. llama.cpp/ggml/src/ggml-opencl/kernels/softmax_4_f16.cl +108 -0
  38. llama.cpp/ggml/src/ggml-opencl/kernels/softmax_4_f32.cl +108 -0
  39. llama.cpp/ggml/src/ggml-opencl/kernels/softmax_f16.cl +107 -0
  40. llama.cpp/ggml/src/ggml-opencl/kernels/softmax_f32.cl +107 -0
  41. llama.cpp/ggml/src/ggml-opencl/kernels/softplus.cl +116 -0
  42. llama.cpp/ggml/src/ggml-opencl/kernels/solve_tri.cl +51 -0
  43. llama.cpp/ggml/src/ggml-opencl/kernels/sqr.cl +53 -0
  44. llama.cpp/ggml/src/ggml-opencl/kernels/sqrt.cl +53 -0
  45. llama.cpp/ggml/src/ggml-opencl/kernels/ssm_conv.cl +77 -0
  46. llama.cpp/ggml/src/ggml-opencl/kernels/sub.cl +138 -0
  47. llama.cpp/ggml/src/ggml-opencl/kernels/sum_rows.cl +140 -0
  48. llama.cpp/ggml/src/ggml-opencl/kernels/tanh.cl +109 -0
  49. llama.cpp/ggml/src/ggml-opencl/kernels/transpose.cl +143 -0
  50. llama.cpp/ggml/src/ggml-opencl/kernels/tri.cl +32 -0
llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32.cl ADDED
@@ -0,0 +1,164 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #pragma OPENCL EXTENSION cl_khr_fp16 : enable
2
+
3
+ #ifdef cl_intel_subgroups
4
+ #pragma OPENCL EXTENSION cl_intel_subgroups : enable
5
+ #else
6
+ #pragma OPENCL EXTENSION cl_khr_subgroups : enable
7
+ #endif
8
+
9
+ #ifdef cl_intel_required_subgroup_size
10
+ #pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
11
+ #define INTEL_GPU 1
12
+ #define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
13
+ #define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
14
+ #elif defined(cl_qcom_reqd_sub_group_size)
15
+ #pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
16
+ #define ADRENO_GPU 1
17
+ #define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
18
+ #define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
19
+ #endif
20
+
21
+ #define QK4_NL 32
22
+
23
+ typedef char int8_t;
24
+ typedef uchar uint8_t;
25
+ typedef short int16_t;
26
+ typedef ushort uint16_t;
27
+ typedef int int32_t;
28
+ typedef uint uint32_t;
29
+
30
+ constant float kvalues_iq4nl[16] = {
31
+ -127.f, -104.f, -83.f, -65.f, -49.f, -35.f, -22.f, -10.f,
32
+ 1.f, 13.f, 25.f, 38.f, 53.f, 69.f, 89.f, 113.f
33
+ };
34
+
35
+ //------------------------------------------------------------------------------
36
+ // block_iq4_nl
37
+ //------------------------------------------------------------------------------
38
+ struct block_iq4_nl
39
+ {
40
+ half d;
41
+ uint8_t qs[QK4_NL / 2];
42
+ };
43
+
44
+ //------------------------------------------------------------------------------
45
+ // mul_vec_q_n_f32
46
+ //------------------------------------------------------------------------------
47
+ // Compute inner product between half a block of iq4_nl and 16 floats (yl).
48
+ // il indicates where the quants begin (0 or 8).
49
+ inline float block_iq4_nl_dot_y(
50
+ global struct block_iq4_nl * qb_curr,
51
+ private float * yl,
52
+ int il
53
+ ) {
54
+ float d = qb_curr->d;
55
+ float acc = 0.f;
56
+ global uchar * qs = qb_curr->qs + il;
57
+ for (int i = 0; i < 8; ++i) {
58
+ acc += yl[i] * kvalues_iq4nl[qs[i] & 0x0F];
59
+ acc += yl[i+8] * kvalues_iq4nl[qs[i] >> 4];
60
+ }
61
+ return d * acc;
62
+ }
63
+
64
+ #ifdef INTEL_GPU
65
+ #define N_DST 4 // each subgroup group works on 4 rows
66
+ #define N_SUBGROUP 1 // number of subgroups in a thread group
67
+ #define N_SUBGROUP_SIZE 16 // assuming subgroup size is 16
68
+ #elif defined (ADRENO_GPU)
69
+ #define N_DST 4
70
+ #define N_SUBGROUP 1
71
+ #define N_SUBGROUP_SIZE 64
72
+ #endif
73
+
74
+ inline void mul_vec_q_n_f32(
75
+ global void * src0,
76
+ global float * src1,
77
+ global float * dst,
78
+ int ne00,
79
+ int ne01,
80
+ int ne02,
81
+ int ne10,
82
+ int ne12,
83
+ int ne0,
84
+ int ne1,
85
+ int r2,
86
+ int r3
87
+ ) {
88
+
89
+ const ulong nb = ne00/QK4_NL;
90
+
91
+ int r0 = get_group_id(0);
92
+ int r1 = get_group_id(1);
93
+ int im = get_group_id(2);
94
+
95
+ int first_row = (r0 * N_SUBGROUP + get_sub_group_id()) * N_DST;
96
+
97
+ int i12 = im%ne12;
98
+ int i13 = im/ne12;
99
+
100
+ ulong offset0 = first_row * nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);
101
+
102
+ global struct block_iq4_nl * x = (global struct block_iq4_nl *) src0 + offset0;
103
+ global float * y = (global float *) src1 + r1*ne10 + im*ne00*ne1;
104
+
105
+ float yl[16]; // src1 vector cache
106
+ float sumf[N_DST]={0.f};
107
+
108
+ int ix = get_sub_group_local_id()/2;
109
+ int il = 8*(get_sub_group_local_id()%2);
110
+
111
+ global float * yb = y + ix * QK4_NL + il;
112
+
113
+ // each thread in a SIMD group deals with half a block.
114
+ for (int ib = ix; ib < nb; ib += N_SUBGROUP_SIZE/2) {
115
+ for (int i = 0; i < 8; ++i) {
116
+ yl[i] = yb[i];
117
+ yl[i+8] = yb[i+16];
118
+ }
119
+
120
+ for (int row = 0; row < N_DST; row++) {
121
+ sumf[row] += block_iq4_nl_dot_y(x+ib+row*nb, yl, il);
122
+ }
123
+
124
+ yb += QK4_NL * (N_SUBGROUP_SIZE/2);
125
+ }
126
+
127
+ float tot[N_DST] = {
128
+ sub_group_reduce_add(sumf[0]), sub_group_reduce_add(sumf[1]),
129
+ sub_group_reduce_add(sumf[2]), sub_group_reduce_add(sumf[3])};
130
+ for (int row = 0; row < N_DST; ++row) {
131
+ if (get_sub_group_local_id() == 0 && first_row + row < ne01) {
132
+ dst[r1*ne0 + im*ne0*ne1 + first_row + row] = tot[row];
133
+ }
134
+ }
135
+ }
136
+
137
+ #ifdef INTEL_GPU
138
+ REQD_SUBGROUP_SIZE_16
139
+ #elif defined (ADRENO_GPU)
140
+ REQD_SUBGROUP_SIZE_64
141
+ #endif
142
+ kernel void kernel_mul_mv_iq4_nl_f32(
143
+ global void * src0,
144
+ ulong offset0,
145
+ global float * src1,
146
+ ulong offset1,
147
+ global float * dst,
148
+ ulong offsetd,
149
+ int ne00,
150
+ int ne01,
151
+ int ne02,
152
+ int ne10,
153
+ int ne12,
154
+ int ne0,
155
+ int ne1,
156
+ int r2,
157
+ int r3
158
+ ) {
159
+ src0 = (global void*)((global char*)src0 + offset0);
160
+ src1 = (global float*)((global char*)src1 + offset1);
161
+ dst = (global float*)((global char*)dst + offsetd);
162
+
163
+ mul_vec_q_n_f32(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3);
164
+ }
llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32_flat.cl ADDED
@@ -0,0 +1,202 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #pragma OPENCL EXTENSION cl_khr_fp16 : enable
2
+
3
+ #ifdef cl_intel_subgroups
4
+ #pragma OPENCL EXTENSION cl_intel_subgroups : enable
5
+ #else
6
+ #pragma OPENCL EXTENSION cl_khr_subgroups : enable
7
+ #endif
8
+
9
+ #ifdef cl_intel_required_subgroup_size
10
+ #pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
11
+ #define INTEL_GPU 1
12
+ #define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
13
+ #define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
14
+ #elif defined(cl_qcom_reqd_sub_group_size)
15
+ #pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
16
+ #define ADRENO_GPU 1
17
+ #define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
18
+ #define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
19
+ #endif
20
+
21
+ #define QK4_NL 32
22
+
23
+ typedef char int8_t;
24
+ typedef uchar uint8_t;
25
+ typedef short int16_t;
26
+ typedef ushort uint16_t;
27
+ typedef int int32_t;
28
+ typedef uint uint32_t;
29
+
30
+ constant float kvalues_iq4nl[16] = {
31
+ -127.f, -104.f, -83.f, -65.f, -49.f, -35.f, -22.f, -10.f,
32
+ 1.f, 13.f, 25.f, 38.f, 53.f, 69.f, 89.f, 113.f
33
+ };
34
+
35
+ //------------------------------------------------------------------------------
36
+ // block_iq4_nl
37
+ //------------------------------------------------------------------------------
38
+ struct block_iq4_nl
39
+ {
40
+ half d;
41
+ uint8_t qs[QK4_NL / 2];
42
+ };
43
+
44
+ // Compute dot product between half a block of iq4_nl quants and activations.
45
+ // x points to the quant bytes, dh points to the scale.
46
+ // yl has 16 activation values: [0..7] for low nibbles, [8..15] for high nibbles.
47
+ // il indicates offset into the quant bytes (0 or 8).
48
+ inline float block_iq4_nl_dot_y_flat(
49
+ global uchar * x,
50
+ global half * dh,
51
+ private float * yl,
52
+ int il
53
+ ) {
54
+ float d = *dh;
55
+ global uchar * qs = x + il;
56
+ float acc = 0.f;
57
+ for (int i = 0; i < 8; ++i) {
58
+ acc += yl[i] * kvalues_iq4nl[qs[i] & 0x0F];
59
+ acc += yl[i+8] * kvalues_iq4nl[qs[i] >> 4];
60
+ }
61
+ return d * acc;
62
+ }
63
+
64
+ #undef N_DST
65
+ #undef N_SIMDGROUP
66
+ #undef N_SIMDWIDTH
67
+
68
+ #ifdef INTEL_GPU
69
+ #define N_DST 8 // each subgroup works on 8 rows
70
+ #define N_SUBGROUP 1 // number of subgroups in a thread group
71
+ #define N_SUBGROUP_SIZE 16 // assuming subgroup size is 16
72
+ #elif defined (ADRENO_GPU)
73
+ #define N_DST 8
74
+ #define N_SUBGROUP 1
75
+ #define N_SUBGROUP_SIZE 64
76
+ #endif
77
+
78
+ inline void mul_vec_q_n_f32_8x_flat(
79
+ global uchar * src0_q,
80
+ global half * src0_d,
81
+ global float * src1,
82
+ global float * dst,
83
+ int ne00,
84
+ int ne01,
85
+ int ne02,
86
+ int ne10,
87
+ int ne12,
88
+ int ne0,
89
+ int ne1,
90
+ int r2,
91
+ int r3
92
+ ) {
93
+ const ulong nb = ne00/QK4_NL;
94
+
95
+ int r0 = get_group_id(0);
96
+ int r1 = get_group_id(1);
97
+ int im = get_group_id(2);
98
+
99
+ int first_row = (r0 * N_SUBGROUP + get_sub_group_id()) * N_DST;
100
+
101
+ int i12 = im%ne12;
102
+ int i13 = im/ne12;
103
+
104
+ // The number of scales is the same as the number of blocks.
105
+ ulong offset0_d = first_row * nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);
106
+ // Each block contains QK4_NL/2 uchars, hence offset for qs is as follows.
107
+ ulong offset0_q = (first_row * nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02)) * QK4_NL/2;
108
+
109
+ global uchar * x = (global uchar *) src0_q + offset0_q;
110
+ global half * d = (global half *) src0_d + offset0_d;
111
+ global float * y = (global float *) src1 + r1*ne10 + im*ne00*ne1;
112
+
113
+ float yl[16];
114
+ float8 sumf = 0.f;
115
+
116
+ int ix = get_sub_group_local_id()/2;
117
+ int il = 8*(get_sub_group_local_id()%2);
118
+
119
+ global float * yb = y + ix*QK4_NL + il;
120
+
121
+ for (int ib = ix; ib < nb; ib += N_SUBGROUP_SIZE/2) {
122
+ for (int i = 0; i < 8; ++i) {
123
+ yl[i] = yb[i];
124
+ yl[i+8] = yb[i+16];
125
+ }
126
+
127
+ sumf.s0 += block_iq4_nl_dot_y_flat(x + ib*QK4_NL/2 + 0*nb*QK4_NL/2, d + ib + 0*nb, yl, il);
128
+ sumf.s1 += block_iq4_nl_dot_y_flat(x + ib*QK4_NL/2 + 1*nb*QK4_NL/2, d + ib + 1*nb, yl, il);
129
+ sumf.s2 += block_iq4_nl_dot_y_flat(x + ib*QK4_NL/2 + 2*nb*QK4_NL/2, d + ib + 2*nb, yl, il);
130
+ sumf.s3 += block_iq4_nl_dot_y_flat(x + ib*QK4_NL/2 + 3*nb*QK4_NL/2, d + ib + 3*nb, yl, il);
131
+
132
+ sumf.s4 += block_iq4_nl_dot_y_flat(x + ib*QK4_NL/2 + 4*nb*QK4_NL/2, d + ib + 4*nb, yl, il);
133
+ sumf.s5 += block_iq4_nl_dot_y_flat(x + ib*QK4_NL/2 + 5*nb*QK4_NL/2, d + ib + 5*nb, yl, il);
134
+ sumf.s6 += block_iq4_nl_dot_y_flat(x + ib*QK4_NL/2 + 6*nb*QK4_NL/2, d + ib + 6*nb, yl, il);
135
+ sumf.s7 += block_iq4_nl_dot_y_flat(x + ib*QK4_NL/2 + 7*nb*QK4_NL/2, d + ib + 7*nb, yl, il);
136
+
137
+ yb += QK4_NL * (N_SUBGROUP_SIZE/2);
138
+ }
139
+
140
+ float8 tot = (float8)(
141
+ sub_group_reduce_add(sumf.s0), sub_group_reduce_add(sumf.s1),
142
+ sub_group_reduce_add(sumf.s2), sub_group_reduce_add(sumf.s3),
143
+ sub_group_reduce_add(sumf.s4), sub_group_reduce_add(sumf.s5),
144
+ sub_group_reduce_add(sumf.s6), sub_group_reduce_add(sumf.s7)
145
+ );
146
+
147
+ if (get_sub_group_local_id() == 0) {
148
+ if (first_row + 0 < ne01) {
149
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 0] = tot.s0;
150
+ }
151
+ if (first_row + 1 < ne01) {
152
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 1] = tot.s1;
153
+ }
154
+ if (first_row + 2 < ne01) {
155
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 2] = tot.s2;
156
+ }
157
+ if (first_row + 3 < ne01) {
158
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 3] = tot.s3;
159
+ }
160
+
161
+ if (first_row + 4 < ne01) {
162
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 4] = tot.s4;
163
+ }
164
+ if (first_row + 5 < ne01) {
165
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 5] = tot.s5;
166
+ }
167
+ if (first_row + 6 < ne01) {
168
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 6] = tot.s6;
169
+ }
170
+ if (first_row + 7 < ne01) {
171
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 7] = tot.s7;
172
+ }
173
+ }
174
+ }
175
+
176
+ #ifdef INTEL_GPU
177
+ REQD_SUBGROUP_SIZE_16
178
+ #elif defined (ADRENO_GPU)
179
+ REQD_SUBGROUP_SIZE_64
180
+ #endif
181
+ kernel void kernel_mul_mv_iq4_nl_f32_flat(
182
+ global uchar * src0_q,
183
+ global half * src0_d,
184
+ global float * src1,
185
+ ulong offset1,
186
+ global float * dst,
187
+ ulong offsetd,
188
+ int ne00,
189
+ int ne01,
190
+ int ne02,
191
+ int ne10,
192
+ int ne12,
193
+ int ne0,
194
+ int ne1,
195
+ int r2,
196
+ int r3
197
+ ) {
198
+ src1 = (global float*)((global char*)src1 + offset1);
199
+ dst = (global float*)((global char*)dst + offsetd);
200
+
201
+ mul_vec_q_n_f32_8x_flat(src0_q, src0_d, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3);
202
+ }
llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32.cl ADDED
@@ -0,0 +1,144 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #pragma OPENCL EXTENSION cl_khr_fp16 : enable
2
+
3
+ #ifdef cl_intel_subgroups
4
+ #pragma OPENCL EXTENSION cl_intel_subgroups : enable
5
+ #else
6
+ #pragma OPENCL EXTENSION cl_khr_subgroups : enable
7
+ #endif
8
+
9
+ #ifdef cl_intel_required_subgroup_size
10
+ #pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
11
+ #define INTEL_GPU 1
12
+ #define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
13
+ #define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
14
+ #elif defined(cl_qcom_reqd_sub_group_size)
15
+ #pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
16
+ #define ADRENO_GPU 1
17
+ #define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
18
+ #define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
19
+ #endif
20
+
21
+ #define QK_MXFP4 32
22
+ typedef struct {
23
+ uchar e; // E8M0
24
+ uchar qs[QK_MXFP4/2];
25
+ } block_mxfp4;
26
+
27
+ constant static float kvalues_mxfp4_f[16] = {
28
+ 0, .5f, 1.f, 1.5f, 2.f, 3.f, 4.f, 6.f, -0, -.5f, -1.f, -1.5f, -2.f, -3.f, -4.f, -6.f
29
+ };
30
+
31
+ static inline float e8m0_to_fp32(uchar x) {
32
+ int bits;
33
+
34
+ if (x == 0) {
35
+ bits = 0x00400000;
36
+ } else {
37
+ bits = (uint) x << 23;
38
+ }
39
+
40
+ return as_float(bits);
41
+ }
42
+
43
+ #ifdef INTEL_GPU
44
+ #define N_R0_MXFP4 2 // number of rows each subgroup works on
45
+ #define N_SG_MXFP4 2 // number of subgroups in a work group
46
+ #define N_SIMDWIDTH 16 // subgroup size
47
+ #elif defined (ADRENO_GPU)
48
+ #define N_R0_MXFP4 2
49
+ #define N_SG_MXFP4 2
50
+ #define N_SIMDWIDTH 64
51
+ #endif
52
+
53
+ #ifdef INTEL_GPU
54
+ REQD_SUBGROUP_SIZE_16
55
+ #elif defined (ADRENO_GPU)
56
+ REQD_SUBGROUP_SIZE_64
57
+ #endif
58
+ kernel void kernel_mul_mv_mxfp4_f32(
59
+ global char * src0,
60
+ ulong offset0,
61
+ global char * src1,
62
+ ulong offset1,
63
+ global char * dst,
64
+ ulong offsetd,
65
+ int ne00,
66
+ ulong nb01,
67
+ ulong nb02,
68
+ ulong nb03,
69
+ int ne12,
70
+ ulong nb11,
71
+ ulong nb12,
72
+ ulong nb13,
73
+ int ne0,
74
+ int ne1,
75
+ int r2,
76
+ int r3,
77
+ local char * shmem
78
+ ) {
79
+ src0 = (global char*)((global char*)src0 + offset0);
80
+ src1 = (global char*)((global char*)src1 + offset1);
81
+ dst = (global char*)((global char*)dst + offsetd);
82
+
83
+ local float * shmem_f32 = (local float *) shmem;
84
+ int nb = ne00/QK_MXFP4;
85
+
86
+ int r0 = get_group_id(0);
87
+ int r1 = get_group_id(1);
88
+ int im = get_group_id(2);
89
+
90
+ int first_row = (r0 * N_SG_MXFP4 + get_sub_group_id()) * N_R0_MXFP4;
91
+
92
+ uint i12 = im%ne12;
93
+ uint i13 = im/ne12;
94
+
95
+ ulong offset_src0 = first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03;
96
+ ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13;
97
+
98
+ global block_mxfp4 * x = (global block_mxfp4 *) (src0 + offset_src0);
99
+ global float * y = (global float *) (src1 + offset_src1);
100
+
101
+ const short ix = get_sub_group_local_id()/2; // 0...15
102
+ const short it = get_sub_group_local_id()%2; // 0 or 1
103
+
104
+ shmem_f32[get_sub_group_local_id()] = kvalues_mxfp4_f[get_sub_group_local_id()%16];
105
+ barrier(CLK_LOCAL_MEM_FENCE);
106
+
107
+ float4 yl[4];
108
+ float sumf[N_R0_MXFP4] = {0.f};
109
+
110
+ global float * yb = y + ix * QK_MXFP4 + it * 8;
111
+
112
+ for (int ib = ix; ib < nb; ib += N_SIMDWIDTH/2) {
113
+ global float4 * y4 = (global float4 *)yb;
114
+ yl[0] = y4[0];
115
+ yl[1] = y4[4];
116
+ yl[2] = y4[1];
117
+ yl[3] = y4[5];
118
+
119
+ for (short row = 0; row < N_R0_MXFP4; row++) {
120
+ global block_mxfp4 * xb = x + row*nb + ib;
121
+ global uchar * q2 = (global uchar *)(xb->qs + 8*it);
122
+
123
+ float4 acc1 = yl[0]*(float4)(shmem_f32[q2[0] & 0x0F], shmem_f32[q2[1] & 0x0F], shmem_f32[q2[2] & 0x0F], shmem_f32[q2[3] & 0x0F]);
124
+ float4 acc2 = yl[1]*(float4)(shmem_f32[q2[0] >> 4 ], shmem_f32[q2[1] >> 4 ], shmem_f32[q2[2] >> 4 ], shmem_f32[q2[3] >> 4 ]);
125
+ float4 acc3 = yl[2]*(float4)(shmem_f32[q2[4] & 0x0F], shmem_f32[q2[5] & 0x0F], shmem_f32[q2[6] & 0x0F], shmem_f32[q2[7] & 0x0F]);
126
+ float4 acc4 = yl[3]*(float4)(shmem_f32[q2[4] >> 4 ], shmem_f32[q2[5] >> 4 ], shmem_f32[q2[6] >> 4 ], shmem_f32[q2[7] >> 4 ]);
127
+
128
+ acc1 = (acc1 + acc3) + (acc2 + acc4);
129
+
130
+ sumf[row] += e8m0_to_fp32(xb->e) * ((acc1.s0 + acc1.s1) + (acc1.s2 + acc1.s3));
131
+ }
132
+
133
+ yb += (N_SIMDWIDTH/2) * QK_MXFP4;
134
+ }
135
+
136
+ global float * dst_f32 = (global float *) dst + (ulong)im*ne0*ne1 + (ulong)r1*ne0;
137
+
138
+ for (int row = 0; row < N_R0_MXFP4 && first_row + row < ne0; ++row) {
139
+ float sum_all = sub_group_reduce_add(sumf[row]);
140
+ if (get_sub_group_local_id() == 0) {
141
+ dst_f32[first_row + row] = sum_all;
142
+ }
143
+ }
144
+ }
llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32_flat.cl ADDED
@@ -0,0 +1,167 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #pragma OPENCL EXTENSION cl_khr_fp16 : enable
2
+
3
+ #ifdef cl_intel_subgroups
4
+ #pragma OPENCL EXTENSION cl_intel_subgroups : enable
5
+ #else
6
+ #pragma OPENCL EXTENSION cl_khr_subgroups : enable
7
+ #endif
8
+
9
+ #ifdef cl_intel_required_subgroup_size
10
+ #pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
11
+ #define INTEL_GPU 1
12
+ #define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
13
+ #define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
14
+ #elif defined(cl_qcom_reqd_sub_group_size)
15
+ #pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
16
+ #define ADRENO_GPU 1
17
+ #define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
18
+ #define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
19
+ #endif
20
+
21
+ #define QK_MXFP4 32
22
+
23
+ static inline half4 mxfp4_to_fp16_packed(ushort fp4x4) {
24
+ ushort2 fp16_packed_a, fp16_packed_b, bias_a, bias_b, sign_a, sign_b;
25
+ fp16_packed_a.lo = (fp4x4 << 9) & 0x0E00;
26
+ fp16_packed_a.hi = (fp4x4 << 5) & 0x0E00;
27
+ fp16_packed_b.lo = (fp4x4 << 1) & 0x0E00;
28
+ fp16_packed_b.hi = (fp4x4 >> 3) & 0x0E00;
29
+
30
+ bias_a.lo = (fp16_packed_a.lo == 0) ? 0x0 : 0x3800;
31
+ bias_a.hi = (fp16_packed_a.hi == 0) ? 0x0 : 0x3800;
32
+ bias_b.lo = (fp16_packed_b.lo == 0) ? 0x0 : 0x3800;
33
+ bias_b.hi = (fp16_packed_b.hi == 0) ? 0x0 : 0x3800;
34
+
35
+ fp16_packed_a.lo = (fp16_packed_a.lo == 0x0200) ? 0x0 : fp16_packed_a.lo;
36
+ fp16_packed_a.hi = (fp16_packed_a.hi == 0x0200) ? 0x0 : fp16_packed_a.hi;
37
+ fp16_packed_b.lo = (fp16_packed_b.lo == 0x0200) ? 0x0 : fp16_packed_b.lo;
38
+ fp16_packed_b.hi = (fp16_packed_b.hi == 0x0200) ? 0x0 : fp16_packed_b.hi;
39
+
40
+ sign_a.lo = (fp4x4 << 12) & 0x8000;
41
+ sign_a.hi = (fp4x4 << 8) & 0x8000;
42
+ sign_b.lo = (fp4x4 << 4) & 0x8000;
43
+ sign_b.hi = fp4x4 & 0x8000;
44
+
45
+ fp16_packed_a = sign_a + bias_a + fp16_packed_a;
46
+ fp16_packed_b = sign_b + bias_b + fp16_packed_b;
47
+
48
+ return as_half4((ushort4)(fp16_packed_a, fp16_packed_b));
49
+ }
50
+
51
+ static inline float e8m0_to_fp32(uchar x) {
52
+ int bits;
53
+ bits = (x == 0) ? 0x00400000 : ((uint) x << 23);
54
+ return as_float(bits);
55
+ }
56
+
57
+ #ifdef INTEL_GPU
58
+ #define N_R0_MXFP4 2 // number of rows each subgroup works on
59
+ #define N_SG_MXFP4 2 // number of subgroups in a work group
60
+ #define N_SIMDWIDTH 16 // subgroup size
61
+ #elif defined (ADRENO_GPU)
62
+ #define N_R0_MXFP4 2
63
+ #define N_SG_MXFP4 2
64
+ #define N_SIMDWIDTH 64
65
+ #define SRC0Q_IMG
66
+ #endif
67
+
68
+ #ifdef INTEL_GPU
69
+ REQD_SUBGROUP_SIZE_16
70
+ #elif defined (ADRENO_GPU)
71
+ REQD_SUBGROUP_SIZE_64
72
+ #endif
73
+ kernel void kernel_mul_mv_mxfp4_f32_flat(
74
+ #ifdef SRC0Q_IMG
75
+ __read_only image1d_buffer_t src0_q,
76
+ #else
77
+ global uchar * src0_q,
78
+ #endif
79
+ global uchar * src0_e,
80
+ global uchar * src1,
81
+ ulong offset1,
82
+ global uchar * dst,
83
+ ulong offsetd,
84
+ int ne00,
85
+ ulong nb01,
86
+ ulong nb02,
87
+ ulong nb03,
88
+ int ne12,
89
+ ulong nb11,
90
+ ulong nb12,
91
+ ulong nb13,
92
+ int ne0,
93
+ int ne1,
94
+ int r2,
95
+ int r3
96
+ ) {
97
+ src1 = src1 + offset1;
98
+ dst = dst + offsetd;
99
+
100
+ int nb = ne00 / QK_MXFP4;
101
+
102
+ int r0 = get_group_id(0);
103
+ int r1 = get_group_id(1);
104
+ int im = get_group_id(2);
105
+
106
+ int first_row = (r0 * N_SG_MXFP4 + get_sub_group_id()) * N_R0_MXFP4;
107
+
108
+ uint i12 = im % ne12;
109
+ uint i13 = im / ne12;
110
+
111
+ uint offset_src0 = first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03;
112
+ // 17 = sizeof(block_mxfp4)
113
+ offset_src0 /= 17;
114
+ #ifdef SRC0Q_IMG
115
+ ulong offset_q = offset_src0;
116
+ #else
117
+ global uchar16 * x_q = (global uchar16 *)(src0_q) + offset_src0;
118
+ #endif
119
+ global uchar * x_e = src0_e + offset_src0;
120
+
121
+ ulong offset_src1 = r1 * nb11 + i12 * nb12 + i13 * nb13;
122
+ global float * y = (global float *)(src1 + offset_src1);
123
+
124
+ const short ix = get_sub_group_local_id() >> 1; // 0...15
125
+ const short it = get_sub_group_local_id() & 1; // 0 or 1
126
+
127
+ float sumf[N_R0_MXFP4] = {0.f};
128
+
129
+ global float * yb = y + ix * QK_MXFP4 + it * 8;
130
+
131
+ for (int ib = ix; ib < nb; ib += N_SIMDWIDTH/2) {
132
+ global float4 * y4 = (global float4 *)yb;
133
+
134
+ #pragma unroll
135
+ for (short row = 0; row < N_R0_MXFP4; row++) {
136
+ uchar xb_e = x_e[row * nb + ib];
137
+ #ifdef SRC0Q_IMG
138
+ ushort4 xb_q = as_ushort4(read_imageui(src0_q, (offset_q + row * nb + ib) * 2 + it).xy);
139
+ #else
140
+ ushort4 xb_q = vload4(0, (global ushort *)((global uchar *)(x_q + row * nb + ib) + 8 * it));
141
+ #endif
142
+
143
+ half4 fp16x4_0 = mxfp4_to_fp16_packed(xb_q.s0);
144
+ half4 fp16x4_1 = mxfp4_to_fp16_packed(xb_q.s1);
145
+ float4 acc1 = y4[0] * (float4)(fp16x4_0.s0, fp16x4_0.s2, fp16x4_1.s0, fp16x4_1.s2);
146
+ acc1 += y4[4] * (float4)(fp16x4_0.s1, fp16x4_0.s3, fp16x4_1.s1, fp16x4_1.s3);
147
+
148
+ fp16x4_0 = mxfp4_to_fp16_packed(xb_q.s2);
149
+ fp16x4_1 = mxfp4_to_fp16_packed(xb_q.s3);
150
+ acc1 += y4[1] * (float4)(fp16x4_0.s0, fp16x4_0.s2, fp16x4_1.s0, fp16x4_1.s2);
151
+ acc1 += y4[5] * (float4)(fp16x4_0.s1, fp16x4_0.s3, fp16x4_1.s1, fp16x4_1.s3);
152
+
153
+ sumf[row] += e8m0_to_fp32(xb_e) * ((acc1.s0 + acc1.s1) + (acc1.s2 + acc1.s3));
154
+ }
155
+
156
+ yb += (N_SIMDWIDTH/2) * QK_MXFP4;
157
+ }
158
+
159
+ global float * dst_f32 = (global float *) dst + (ulong)im*ne0*ne1 + (ulong)r1*ne0;
160
+
161
+ for (int row = 0; row < N_R0_MXFP4 && first_row + row < ne0; ++row) {
162
+ float sum_all = sub_group_reduce_add(sumf[row]);
163
+ if (get_sub_group_local_id() == 0) {
164
+ dst_f32[first_row + row] = sum_all;
165
+ }
166
+ }
167
+ }
llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32.cl ADDED
@@ -0,0 +1,141 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #pragma OPENCL EXTENSION cl_khr_fp16 : enable
2
+
3
+ #ifdef cl_intel_subgroups
4
+ #pragma OPENCL EXTENSION cl_intel_subgroups : enable
5
+ #else
6
+ #pragma OPENCL EXTENSION cl_khr_subgroups : enable
7
+ #endif
8
+
9
+ #ifdef cl_intel_required_subgroup_size
10
+ #pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
11
+ #define INTEL_GPU 1
12
+ #define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
13
+ #define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
14
+ #elif defined(cl_qcom_reqd_sub_group_size)
15
+ #pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
16
+ #define ADRENO_GPU 1
17
+ #define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
18
+ #define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
19
+ #endif
20
+
21
+ #define QK1_0 128
22
+ typedef struct {
23
+ half d;
24
+ uchar qs[QK1_0/8];
25
+ } block_q1_0;
26
+
27
+ #define NB_Q1_0 16
28
+
29
+ #ifdef INTEL_GPU
30
+ #define N_R0_Q1_0 4 // number of rows each subgroup works on
31
+ #define N_SG_Q1_0 2 // number of subgroups in a work group
32
+ #define N_SIMDWIDTH 16 // subgroup size
33
+ #elif defined (ADRENO_GPU)
34
+ #define N_R0_Q1_0 4
35
+ #define N_SG_Q1_0 2
36
+ #define N_SIMDWIDTH 64
37
+ #endif
38
+
39
+ inline float block_q_1_0_dot_y(global block_q1_0 * qb, float sumy, float yl[NB_Q1_0], short il) {
40
+ global uchar * qs = qb->qs + il*2;
41
+ uint b0 = qs[0];
42
+ uint b1 = qs[1];
43
+
44
+ float acc = 0.f;
45
+ acc += yl[ 0]*(float)((b0 >> 0) & 1) + yl[ 1]*(float)((b0 >> 1) & 1);
46
+ acc += yl[ 2]*(float)((b0 >> 2) & 1) + yl[ 3]*(float)((b0 >> 3) & 1);
47
+ acc += yl[ 4]*(float)((b0 >> 4) & 1) + yl[ 5]*(float)((b0 >> 5) & 1);
48
+ acc += yl[ 6]*(float)((b0 >> 6) & 1) + yl[ 7]*(float)((b0 >> 7) & 1);
49
+
50
+ acc += yl[ 8]*(float)((b1 >> 0) & 1) + yl[ 9]*(float)((b1 >> 1) & 1);
51
+ acc += yl[10]*(float)((b1 >> 2) & 1) + yl[11]*(float)((b1 >> 3) & 1);
52
+ acc += yl[12]*(float)((b1 >> 4) & 1) + yl[13]*(float)((b1 >> 5) & 1);
53
+ acc += yl[14]*(float)((b1 >> 6) & 1) + yl[15]*(float)((b1 >> 7) & 1);
54
+
55
+ return qb->d * (2.0f*acc - sumy);
56
+ }
57
+
58
+ #ifdef INTEL_GPU
59
+ REQD_SUBGROUP_SIZE_16
60
+ #elif defined (ADRENO_GPU)
61
+ REQD_SUBGROUP_SIZE_64
62
+ #endif
63
+ kernel void kernel_mul_mv_q1_0_f32(
64
+ global char * src0,
65
+ ulong offset0,
66
+ global char * src1,
67
+ ulong offset1,
68
+ global char * dst,
69
+ ulong offsetd,
70
+ int ne00,
71
+ int ne01,
72
+ ulong nb01,
73
+ ulong nb02,
74
+ ulong nb03,
75
+ int ne12,
76
+ ulong nb11,
77
+ ulong nb12,
78
+ ulong nb13,
79
+ int ne0,
80
+ int ne1,
81
+ int r2,
82
+ int r3
83
+ ) {
84
+ src0 = (global char*)((global char*)src0 + offset0);
85
+ src1 = (global char*)((global char*)src1 + offset1);
86
+ dst = (global char*)((global char*)dst + offsetd);
87
+
88
+ int nb = ne00/QK1_0;
89
+
90
+ int r0 = get_group_id(0);
91
+ int r1 = get_group_id(1);
92
+ int im = get_group_id(2);
93
+
94
+ int first_row = (r0*N_SG_Q1_0 + get_sub_group_id()) * N_R0_Q1_0;
95
+
96
+ uint i12 = im%ne12;
97
+ uint i13 = im/ne12;
98
+
99
+ ulong offset_src1 = r1*nb11 + i12*nb12 + i13*nb13;
100
+ global float * y = (global float *) (src1 + offset_src1);
101
+
102
+ // pointers to src0 rows
103
+ global block_q1_0 * ax[N_R0_Q1_0];
104
+ for (int row = 0; row < N_R0_Q1_0; ++row) {
105
+ ulong offset_src0 = (first_row + row)*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03;
106
+ ax[row] = (global block_q1_0 *) ((global char *) src0 + offset_src0);
107
+ }
108
+
109
+ float yl[NB_Q1_0];
110
+ float sumf[N_R0_Q1_0] = { 0.f };
111
+
112
+ const short ix = get_sub_group_local_id()/8;
113
+ const short il = get_sub_group_local_id()%8;
114
+
115
+ global float * yb = y + ix*QK1_0 + il*NB_Q1_0;
116
+
117
+ // each thread handles NB_Q1_0 quants at a time
118
+ for (int ib = ix; ib < nb; ib += N_SIMDWIDTH/8) {
119
+ float sumy = 0.f;
120
+ for (short i = 0; i < NB_Q1_0; ++i) {
121
+ yl[i] = yb[i];
122
+ sumy += yb[i];
123
+ }
124
+
125
+ for (short row = 0; row < N_R0_Q1_0; row++) {
126
+ sumf[row] += block_q_1_0_dot_y(ax[row] + ib, sumy, yl, il);
127
+ }
128
+
129
+ yb += N_SIMDWIDTH*NB_Q1_0;
130
+ }
131
+
132
+ global float * dst_f32 = (global float *) dst + (ulong)im*ne0*ne1 + (ulong)r1*ne0;
133
+
134
+ for (int row = 0; row < N_R0_Q1_0; ++row) {
135
+ float tot = sub_group_reduce_add(sumf[row]);
136
+
137
+ if (get_sub_group_local_id() == 0 && first_row + row < ne01) {
138
+ dst_f32[first_row + row] = tot;
139
+ }
140
+ }
141
+ }
llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32_flat.cl ADDED
@@ -0,0 +1,190 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #pragma OPENCL EXTENSION cl_khr_fp16 : enable
2
+
3
+ #ifdef cl_intel_subgroups
4
+ #pragma OPENCL EXTENSION cl_intel_subgroups : enable
5
+ #else
6
+ #pragma OPENCL EXTENSION cl_khr_subgroups : enable
7
+ #endif
8
+
9
+ #ifdef cl_intel_required_subgroup_size
10
+ #pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
11
+ #define INTEL_GPU 1
12
+ #define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
13
+ #define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
14
+ #elif defined(cl_qcom_reqd_sub_group_size)
15
+ #pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
16
+ #define ADRENO_GPU 1
17
+ #define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
18
+ #define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
19
+ #endif
20
+
21
+ #define QK1_0 128
22
+ #define QK1_0_BYTES (QK1_0/8) // 16 quant bytes per block
23
+ #define QK1_0_BLK_BYTES (QK1_0_BYTES + 2) // d + qs in original tensor = 18
24
+
25
+ #define NB_Q1_0 16 // quants handled per thread (two qs bytes)
26
+
27
+ #ifdef INTEL_GPU
28
+ #define N_R0_Q1_0 4 // number of rows each subgroup works on
29
+ #define N_SG_Q1_0 2 // number of subgroups in a work group
30
+ #define N_SIMDWIDTH 16 // subgroup size
31
+ #elif defined (ADRENO_GPU)
32
+ #define N_R0_Q1_0 4
33
+ #define N_SG_Q1_0 2
34
+ #define N_SIMDWIDTH 64
35
+ #endif
36
+
37
+ #ifdef INTEL_GPU
38
+ REQD_SUBGROUP_SIZE_16
39
+ #elif defined (ADRENO_GPU)
40
+ REQD_SUBGROUP_SIZE_64
41
+ #endif
42
+ kernel void kernel_mul_mv_q1_0_f32_flat(
43
+ global char * src0_q,
44
+ global half * src0_d,
45
+ global char * src1,
46
+ ulong offset1,
47
+ global char * dst,
48
+ ulong offsetd,
49
+ int ne00,
50
+ int ne01,
51
+ ulong nb01,
52
+ ulong nb02,
53
+ ulong nb03,
54
+ int ne12,
55
+ ulong nb11,
56
+ ulong nb12,
57
+ ulong nb13,
58
+ int ne0,
59
+ int ne1,
60
+ int r2,
61
+ int r3
62
+ ) {
63
+ src1 = (global char*)((global char*)src1 + offset1);
64
+ dst = (global char*)((global char*)dst + offsetd);
65
+
66
+ int nb = ne00/QK1_0;
67
+
68
+ int r0 = get_group_id(0);
69
+ int r1 = get_group_id(1);
70
+ int im = get_group_id(2);
71
+
72
+ int first_row = (r0*N_SG_Q1_0 + get_sub_group_id()) * N_R0_Q1_0;
73
+
74
+ uint i12 = im%ne12;
75
+ uint i13 = im/ne12;
76
+
77
+ ulong offset_src1 = r1*nb11 + i12*nb12 + i13*nb13;
78
+ global float * y = (global float *) (src1 + offset_src1);
79
+
80
+ // pointers to src0 rows (flat: q bytes + scales)
81
+ uint offset_src0_base = first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03;
82
+
83
+ global uchar * ax0, * ax1, * ax2, * ax3;
84
+ global half * ad0, * ad1, * ad2, * ad3;
85
+ uint offset_src0;
86
+
87
+ offset_src0 = (offset_src0_base + 0*nb01) / QK1_0_BLK_BYTES;
88
+ ax0 = (global uchar *) ((global char *) src0_q + offset_src0*QK1_0_BYTES);
89
+ ad0 = (global half *) ((global char *) src0_d + offset_src0*sizeof(half));
90
+
91
+ offset_src0 = (offset_src0_base + 1*nb01) / QK1_0_BLK_BYTES;
92
+ ax1 = (global uchar *) ((global char *) src0_q + offset_src0*QK1_0_BYTES);
93
+ ad1 = (global half *) ((global char *) src0_d + offset_src0*sizeof(half));
94
+
95
+ offset_src0 = (offset_src0_base + 2*nb01) / QK1_0_BLK_BYTES;
96
+ ax2 = (global uchar *) ((global char *) src0_q + offset_src0*QK1_0_BYTES);
97
+ ad2 = (global half *) ((global char *) src0_d + offset_src0*sizeof(half));
98
+
99
+ offset_src0 = (offset_src0_base + 3*nb01) / QK1_0_BLK_BYTES;
100
+ ax3 = (global uchar *) ((global char *) src0_q + offset_src0*QK1_0_BYTES);
101
+ ad3 = (global half *) ((global char *) src0_d + offset_src0*sizeof(half));
102
+
103
+ const short ix = get_sub_group_local_id()/8;
104
+ const short il = get_sub_group_local_id()%8;
105
+
106
+ global float * yb = y + ix*QK1_0 + il*NB_Q1_0;
107
+
108
+ float8 yl_lo;
109
+ float8 yl_hi;
110
+ float4 sumf = 0.f;
111
+
112
+ // each thread handles NB_Q1_0 = 16 quants (two qs bytes) at a time
113
+ for (int ib = ix; ib < nb; ib += N_SIMDWIDTH/8) {
114
+ yl_lo = vload8(0, yb);
115
+ yl_hi = vload8(0, yb + 8);
116
+ float sumy = yl_lo.s0 + yl_lo.s1 + yl_lo.s2 + yl_lo.s3
117
+ + yl_lo.s4 + yl_lo.s5 + yl_lo.s6 + yl_lo.s7
118
+ + yl_hi.s0 + yl_hi.s1 + yl_hi.s2 + yl_hi.s3
119
+ + yl_hi.s4 + yl_hi.s5 + yl_hi.s6 + yl_hi.s7;
120
+
121
+ uint b0, b1;
122
+ float acc;
123
+
124
+ b0 = ax0[ib*QK1_0_BYTES + il*2 + 0];
125
+ b1 = ax0[ib*QK1_0_BYTES + il*2 + 1];
126
+ acc = yl_lo.s0*(float)((b0 >> 0) & 1) + yl_lo.s1*(float)((b0 >> 1) & 1)
127
+ + yl_lo.s2*(float)((b0 >> 2) & 1) + yl_lo.s3*(float)((b0 >> 3) & 1)
128
+ + yl_lo.s4*(float)((b0 >> 4) & 1) + yl_lo.s5*(float)((b0 >> 5) & 1)
129
+ + yl_lo.s6*(float)((b0 >> 6) & 1) + yl_lo.s7*(float)((b0 >> 7) & 1)
130
+ + yl_hi.s0*(float)((b1 >> 0) & 1) + yl_hi.s1*(float)((b1 >> 1) & 1)
131
+ + yl_hi.s2*(float)((b1 >> 2) & 1) + yl_hi.s3*(float)((b1 >> 3) & 1)
132
+ + yl_hi.s4*(float)((b1 >> 4) & 1) + yl_hi.s5*(float)((b1 >> 5) & 1)
133
+ + yl_hi.s6*(float)((b1 >> 6) & 1) + yl_hi.s7*(float)((b1 >> 7) & 1);
134
+ sumf.s0 += (float)ad0[ib] * (2.0f*acc - sumy);
135
+
136
+ b0 = ax1[ib*QK1_0_BYTES + il*2 + 0];
137
+ b1 = ax1[ib*QK1_0_BYTES + il*2 + 1];
138
+ acc = yl_lo.s0*(float)((b0 >> 0) & 1) + yl_lo.s1*(float)((b0 >> 1) & 1)
139
+ + yl_lo.s2*(float)((b0 >> 2) & 1) + yl_lo.s3*(float)((b0 >> 3) & 1)
140
+ + yl_lo.s4*(float)((b0 >> 4) & 1) + yl_lo.s5*(float)((b0 >> 5) & 1)
141
+ + yl_lo.s6*(float)((b0 >> 6) & 1) + yl_lo.s7*(float)((b0 >> 7) & 1)
142
+ + yl_hi.s0*(float)((b1 >> 0) & 1) + yl_hi.s1*(float)((b1 >> 1) & 1)
143
+ + yl_hi.s2*(float)((b1 >> 2) & 1) + yl_hi.s3*(float)((b1 >> 3) & 1)
144
+ + yl_hi.s4*(float)((b1 >> 4) & 1) + yl_hi.s5*(float)((b1 >> 5) & 1)
145
+ + yl_hi.s6*(float)((b1 >> 6) & 1) + yl_hi.s7*(float)((b1 >> 7) & 1);
146
+ sumf.s1 += (float)ad1[ib] * (2.0f*acc - sumy);
147
+
148
+ b0 = ax2[ib*QK1_0_BYTES + il*2 + 0];
149
+ b1 = ax2[ib*QK1_0_BYTES + il*2 + 1];
150
+ acc = yl_lo.s0*(float)((b0 >> 0) & 1) + yl_lo.s1*(float)((b0 >> 1) & 1)
151
+ + yl_lo.s2*(float)((b0 >> 2) & 1) + yl_lo.s3*(float)((b0 >> 3) & 1)
152
+ + yl_lo.s4*(float)((b0 >> 4) & 1) + yl_lo.s5*(float)((b0 >> 5) & 1)
153
+ + yl_lo.s6*(float)((b0 >> 6) & 1) + yl_lo.s7*(float)((b0 >> 7) & 1)
154
+ + yl_hi.s0*(float)((b1 >> 0) & 1) + yl_hi.s1*(float)((b1 >> 1) & 1)
155
+ + yl_hi.s2*(float)((b1 >> 2) & 1) + yl_hi.s3*(float)((b1 >> 3) & 1)
156
+ + yl_hi.s4*(float)((b1 >> 4) & 1) + yl_hi.s5*(float)((b1 >> 5) & 1)
157
+ + yl_hi.s6*(float)((b1 >> 6) & 1) + yl_hi.s7*(float)((b1 >> 7) & 1);
158
+ sumf.s2 += (float)ad2[ib] * (2.0f*acc - sumy);
159
+
160
+ b0 = ax3[ib*QK1_0_BYTES + il*2 + 0];
161
+ b1 = ax3[ib*QK1_0_BYTES + il*2 + 1];
162
+ acc = yl_lo.s0*(float)((b0 >> 0) & 1) + yl_lo.s1*(float)((b0 >> 1) & 1)
163
+ + yl_lo.s2*(float)((b0 >> 2) & 1) + yl_lo.s3*(float)((b0 >> 3) & 1)
164
+ + yl_lo.s4*(float)((b0 >> 4) & 1) + yl_lo.s5*(float)((b0 >> 5) & 1)
165
+ + yl_lo.s6*(float)((b0 >> 6) & 1) + yl_lo.s7*(float)((b0 >> 7) & 1)
166
+ + yl_hi.s0*(float)((b1 >> 0) & 1) + yl_hi.s1*(float)((b1 >> 1) & 1)
167
+ + yl_hi.s2*(float)((b1 >> 2) & 1) + yl_hi.s3*(float)((b1 >> 3) & 1)
168
+ + yl_hi.s4*(float)((b1 >> 4) & 1) + yl_hi.s5*(float)((b1 >> 5) & 1)
169
+ + yl_hi.s6*(float)((b1 >> 6) & 1) + yl_hi.s7*(float)((b1 >> 7) & 1);
170
+ sumf.s3 += (float)ad3[ib] * (2.0f*acc - sumy);
171
+
172
+ yb += N_SIMDWIDTH*NB_Q1_0;
173
+ }
174
+
175
+ global float * dst_f32 = (global float *) dst + (ulong)im*ne0*ne1 + (ulong)r1*ne0;
176
+
177
+ float4 tot = (float4)(
178
+ sub_group_reduce_add(sumf.s0),
179
+ sub_group_reduce_add(sumf.s1),
180
+ sub_group_reduce_add(sumf.s2),
181
+ sub_group_reduce_add(sumf.s3)
182
+ );
183
+
184
+ if (get_sub_group_local_id() == 0) {
185
+ if (first_row + 0 < ne01) dst_f32[first_row + 0] = tot.s0;
186
+ if (first_row + 1 < ne01) dst_f32[first_row + 1] = tot.s1;
187
+ if (first_row + 2 < ne01) dst_f32[first_row + 2] = tot.s2;
188
+ if (first_row + 3 < ne01) dst_f32[first_row + 3] = tot.s3;
189
+ }
190
+ }
llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32.cl ADDED
@@ -0,0 +1,192 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #pragma OPENCL EXTENSION cl_khr_fp16 : enable
2
+
3
+ #ifdef cl_intel_subgroups
4
+ #pragma OPENCL EXTENSION cl_intel_subgroups : enable
5
+ #else
6
+ #pragma OPENCL EXTENSION cl_khr_subgroups : enable
7
+ #endif
8
+
9
+ #ifdef cl_intel_required_subgroup_size
10
+ #pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
11
+ #define INTEL_GPU 1
12
+ #define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
13
+ #define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
14
+ #elif defined(cl_qcom_reqd_sub_group_size)
15
+ #pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
16
+ #define ADRENO_GPU 1
17
+ #define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
18
+ #define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
19
+ #endif
20
+
21
+ #define QK4_0 32
22
+ #define QR4_0 2
23
+ #define QK4_1 32
24
+ #define QR4_1 2
25
+ #define QK5_0 32
26
+ #define QR5_0 2
27
+ #define QK5_1 32
28
+ #define QR5_1 2
29
+ #define QK8_0 32
30
+ #define QR8_0 1
31
+ #define QK_K 256
32
+ #define K_QUANTS_PER_ITERATION 2
33
+
34
+ typedef char int8_t;
35
+ typedef uchar uint8_t;
36
+ typedef short int16_t;
37
+ typedef ushort uint16_t;
38
+ typedef int int32_t;
39
+ typedef uint uint32_t;
40
+
41
+ //------------------------------------------------------------------------------
42
+ // block_q4_0
43
+ //------------------------------------------------------------------------------
44
+ struct block_q4_0
45
+ {
46
+ half d;
47
+ uint8_t qs[QK4_0 / 2];
48
+ };
49
+
50
+ //------------------------------------------------------------------------------
51
+ // mul_vec_q_n_f32
52
+ //------------------------------------------------------------------------------
53
+ // function for calculate inner product between half a q4_0 block and 16 floats (yl), sumy is SUM(yl[i])
54
+ // il indicates where the q4 quants begin (0 or QK4_0/4)
55
+ // we assume that the yl's have been multiplied with the appropriate scale factor
56
+ // that corresponds to the missing bit shifts (1, 1/16, 1/256, 1/4096)
57
+ inline float block_q_4_0_dot_y(
58
+ global struct block_q4_0 * qb_curr,
59
+ float sumy,
60
+ private float * yl,
61
+ int il
62
+ ) {
63
+ float d = qb_curr->d;
64
+ float2 acc = 0.f;
65
+ global ushort * qs = ((global ushort *)qb_curr + 1 + il/2);
66
+ for (int i = 0; i < 8; i+=2) {
67
+ acc.s0 += yl[i + 0] * (qs[i / 2] & 0x000F)
68
+ + yl[i + 1] * (qs[i / 2] & 0x0F00);
69
+ acc.s1 += yl[i + 8] * (qs[i / 2] & 0x00F0)
70
+ + yl[i + 9] * (qs[i / 2] & 0xF000);
71
+ }
72
+ return d * (sumy * -8.f + acc.s0 + acc.s1);
73
+ }
74
+
75
+ #ifdef INTEL_GPU
76
+ #define N_DST 4 // each SIMD group works on 4 rows
77
+ #define N_SIMDGROUP 1 // number of SIMD groups in a thread group
78
+ #define N_SIMDWIDTH 16 // assuming SIMD group size is 16
79
+ #elif defined (ADRENO_GPU)
80
+ #define N_DST 4
81
+ #define N_SIMDGROUP 1
82
+ #define N_SIMDWIDTH 64
83
+ #endif
84
+
85
+ inline void mul_vec_q_n_f32(
86
+ global void * src0,
87
+ global float * src1,
88
+ global float * dst,
89
+ int ne00,
90
+ int ne01,
91
+ int ne02,
92
+ int ne10,
93
+ int ne12,
94
+ int ne0,
95
+ int ne1,
96
+ int r2,
97
+ int r3
98
+ ) {
99
+
100
+ const ulong nb = ne00/QK4_0;
101
+
102
+ int r0 = get_group_id(0);
103
+ int r1 = get_group_id(1);
104
+ int im = get_group_id(2);
105
+
106
+ // (r0 * N_SIMDGROUP + get_sub_group_id()) is essenatially the linear global
107
+ // id of a SIMD group in the grid.
108
+ int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;
109
+
110
+ int i12 = im%ne12;
111
+ int i13 = im/ne12;
112
+
113
+ ulong offset0 = first_row * nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);
114
+
115
+ global struct block_q4_0 * x = (global struct block_q4_0 *) src0 + offset0;
116
+ global float * y = (global float *) src1 + r1*ne10 + im*ne00*ne1;
117
+
118
+ float yl[16]; // src1 vector cache
119
+ float sumf[N_DST]={0.f};
120
+
121
+ int ix = get_sub_group_local_id()/2;
122
+ int il = 8*(get_sub_group_local_id()%2);
123
+
124
+ global float * yb = y + ix * QK4_0 + il;
125
+
126
+ // each thread in a SIMD group deals with half a block.
127
+ for (int ib = ix; ib < nb; ib += N_SIMDWIDTH/2) {
128
+ float sumy = 0;
129
+ for (int i = 0; i < 8; i += 2) {
130
+ sumy += yb[i] + yb[i+1];
131
+ yl[i+0] = yb[i+ 0];
132
+ yl[i+1] = yb[i+ 1]/256.f;
133
+ sumy += yb[i+16] + yb[i+17];
134
+ yl[i+8] = yb[i+16]/16.f;
135
+ yl[i+9] = yb[i+17]/4096.f;
136
+ }
137
+
138
+ for (int row = 0; row < N_DST; row++) {
139
+ sumf[row] += block_q_4_0_dot_y(x+ib+row*nb, sumy, yl, il);
140
+ }
141
+
142
+ // One thread in a SIMD group (i.e., subgroup) handles a half block,
143
+ // hence then entire SIMD group handles SIMDWIDTH/2 blocks.
144
+ // y points to the activation matrix (of type float). Therefore for
145
+ // one thread, the # of blocks y should advance is SIMDWIDTH/2 (because
146
+ // SIMDWIDTH/2 blocks are processed by a SIMD group) - in terms of
147
+ // floats, it is QK4_0 * (SIMDWIDTH/2), where QK4_0 is the block size.
148
+ yb += QK4_0 * (N_SIMDWIDTH/2);
149
+ }
150
+
151
+ // The above does not work for Adreno - it produces incorrect results for
152
+ // row = 1, 2, 3 and only row = 0 gives the correct result.
153
+ // If N_DST is changed, the below array must be initialized accordingly.
154
+ // This also seems to perform better on Intel.
155
+ float tot[N_DST] = {
156
+ sub_group_reduce_add(sumf[0]), sub_group_reduce_add(sumf[1]),
157
+ sub_group_reduce_add(sumf[2]), sub_group_reduce_add(sumf[3])};
158
+ for (int row = 0; row < N_DST; ++row) {
159
+ if (get_sub_group_local_id() == 0 && first_row + row < ne01) {
160
+ dst[r1*ne0 + im*ne0*ne1 + first_row + row] = tot[row];
161
+ }
162
+ }
163
+ }
164
+
165
+ #ifdef INTEL_GPU
166
+ REQD_SUBGROUP_SIZE_16
167
+ #elif defined (ADRENO_GPU)
168
+ REQD_SUBGROUP_SIZE_64
169
+ #endif
170
+ kernel void kernel_mul_mat_q4_0_f32(
171
+ global void * src0,
172
+ ulong offset0,
173
+ global float * src1,
174
+ ulong offset1,
175
+ global float * dst,
176
+ ulong offsetd,
177
+ int ne00,
178
+ int ne01,
179
+ int ne02,
180
+ int ne10,
181
+ int ne12,
182
+ int ne0,
183
+ int ne1,
184
+ int r2,
185
+ int r3
186
+ ) {
187
+ src0 = (global void*)((global char*)src0 + offset0);
188
+ src1 = (global float*)((global char*)src1 + offset1);
189
+ dst = (global float*)((global char*)dst + offsetd);
190
+
191
+ mul_vec_q_n_f32(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3);
192
+ }
llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_16x_flat.cl ADDED
@@ -0,0 +1,307 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #pragma OPENCL EXTENSION cl_khr_fp16 : enable
2
+
3
+ #ifdef cl_intel_subgroups
4
+ #pragma OPENCL EXTENSION cl_intel_subgroups : enable
5
+ #else
6
+ #pragma OPENCL EXTENSION cl_khr_subgroups : enable
7
+ #endif
8
+
9
+ #ifdef cl_intel_required_subgroup_size
10
+ #pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
11
+ #define INTEL_GPU 1
12
+ #define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
13
+ #define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
14
+ #elif defined(cl_qcom_reqd_sub_group_size)
15
+ #pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
16
+ #define ADRENO_GPU 1
17
+ #define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
18
+ #define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
19
+ #endif
20
+
21
+ #define QK4_0 32
22
+ #define QR4_0 2
23
+ #define QK4_1 32
24
+ #define QR4_1 2
25
+ #define QK5_0 32
26
+ #define QR5_0 2
27
+ #define QK5_1 32
28
+ #define QR5_1 2
29
+ #define QK8_0 32
30
+ #define QR8_0 1
31
+ #define QK_K 256
32
+ #define K_QUANTS_PER_ITERATION 2
33
+
34
+ typedef char int8_t;
35
+ typedef uchar uint8_t;
36
+ typedef short int16_t;
37
+ typedef ushort uint16_t;
38
+ typedef int int32_t;
39
+ typedef uint uint32_t;
40
+
41
+ //------------------------------------------------------------------------------
42
+ // block_q4_0
43
+ //------------------------------------------------------------------------------
44
+ struct block_q4_0
45
+ {
46
+ half d;
47
+ uint8_t qs[QK4_0 / 2];
48
+ };
49
+
50
+ inline float mm_block_q_4_0_dot_y_flat(
51
+ global uchar * x,
52
+ global half * dh,
53
+ float sumy,
54
+ float16 yl,
55
+ int il
56
+ ) {
57
+ float d = *dh;
58
+ global ushort * qs = ((global ushort *)x + il/2);
59
+ float acc = 0.f;
60
+
61
+ acc += yl.s0 * (qs[0] & 0x000F);
62
+ acc += yl.s1 * (qs[0] & 0x0F00);
63
+ acc += yl.s8 * (qs[0] & 0x00F0);
64
+ acc += yl.s9 * (qs[0] & 0xF000);
65
+
66
+ acc += yl.s2 * (qs[1] & 0x000F);
67
+ acc += yl.s3 * (qs[1] & 0x0F00);
68
+ acc += yl.sa * (qs[1] & 0x00F0);
69
+ acc += yl.sb * (qs[1] & 0xF000);
70
+
71
+ acc += yl.s4 * (qs[2] & 0x000F);
72
+ acc += yl.s5 * (qs[2] & 0x0F00);
73
+ acc += yl.sc * (qs[2] & 0x00F0);
74
+ acc += yl.sd * (qs[2] & 0xF000);
75
+
76
+ acc += yl.s6 * (qs[3] & 0x000F);
77
+ acc += yl.s7 * (qs[3] & 0x0F00);
78
+ acc += yl.se * (qs[3] & 0x00F0);
79
+ acc += yl.sf * (qs[3] & 0xF000);
80
+
81
+ return d * (sumy * -8.f + acc);
82
+ }
83
+
84
+ #ifdef INTEL_GPU
85
+ #define N_DST 16 // each SIMD group works on 8 rows (in weights matrix)
86
+ #define N_SIMDGROUP 1 // number of SIMD groups in a thread group
87
+ #define N_SIMDWIDTH 16 // assuming SIMD group size is 16
88
+ #elif defined (ADRENO_GPU)
89
+ #define N_DST 16
90
+ #define N_SIMDGROUP 1
91
+ #define N_SIMDWIDTH 64
92
+ #endif
93
+ //
94
+ // This variant performs 1d blocking with 16x output.
95
+ // Eeach simdgroup outputs 16 values on `n0` dim (row in the output matrix).
96
+ //
97
+ inline void mul_mat_q_n_f32_1d_16x_flat(
98
+ global uchar * src0_q,
99
+ global half * src0_d,
100
+ global float * src1,
101
+ global float * dst,
102
+ int ne00,
103
+ int ne01,
104
+ int ne02,
105
+ int ne10,
106
+ int ne12,
107
+ int ne0,
108
+ int ne1,
109
+ int r2,
110
+ int r3
111
+ ) {
112
+ const int nb = ne00/QK4_0;
113
+
114
+ int r0 = get_group_id(0);
115
+ int r1 = get_group_id(1);
116
+ int im = get_group_id(2);
117
+
118
+ // (r0 * N_SIMDGROUP + get_sub_group_id()) is the linear global id of
119
+ // a SIMD group in the grid. Each SIMD group produces N_DST values in the
120
+ // result, hence uses nb blocks, i.e., the offset becomes first_row*nb.
121
+ // Currently with llama2 7B, im is always 0.
122
+ // TODO: how to handle im/gqa*(nb*ne0)?
123
+ int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;
124
+
125
+ int i12 = im%ne12;
126
+ int i13 = im/ne12;
127
+
128
+ // The number of scales is the same as the number of blocks.
129
+ ulong offset0_d = first_row * nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);
130
+ // Each block contains QK4_0/2 uchars, hence offset for qs is as follows.
131
+ ulong offset0_q = (first_row * nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02)) * QK4_0/2;
132
+
133
+ global uchar * x = (global uchar *) src0_q + offset0_q;
134
+ global half * d = (global half *) src0_d + offset0_d;
135
+ global float * y = (global float *) src1 + r1*ne10 + im*ne00*ne1;
136
+
137
+ float16 yl;
138
+ float16 sumf = (float16)(0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f,
139
+ 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f);
140
+
141
+ int ix = get_sub_group_local_id()/2;
142
+ int il = 8*(get_sub_group_local_id()%2);
143
+
144
+ global float * yb = y + ix*QK4_0 + il;
145
+
146
+ for (int ib = ix; ib < nb; ib += N_SIMDWIDTH/2) {
147
+ float sumy = 0.f;
148
+
149
+ sumy += yb[0];
150
+ sumy += yb[1];
151
+ sumy += yb[2];
152
+ sumy += yb[3];
153
+ sumy += yb[4];
154
+ sumy += yb[5];
155
+ sumy += yb[6];
156
+ sumy += yb[7];
157
+
158
+ sumy += yb[16];
159
+ sumy += yb[17];
160
+ sumy += yb[18];
161
+ sumy += yb[19];
162
+ sumy += yb[20];
163
+ sumy += yb[21];
164
+ sumy += yb[22];
165
+ sumy += yb[23];
166
+
167
+ yl.s0 = yb[0];
168
+ yl.s1 = yb[1]/256.f;
169
+
170
+ yl.s2 = yb[2];
171
+ yl.s3 = yb[3]/256.f;
172
+
173
+ yl.s4 = yb[4];
174
+ yl.s5 = yb[5]/256.f;
175
+
176
+ yl.s6 = yb[6];
177
+ yl.s7 = yb[7]/256.f;
178
+
179
+ yl.s8 = yb[16]/16.f;
180
+ yl.s9 = yb[17]/4096.f;
181
+
182
+ yl.sa = yb[18]/16.f;
183
+ yl.sb = yb[19]/4096.f;
184
+
185
+ yl.sc = yb[20]/16.f;
186
+ yl.sd = yb[21]/4096.f;
187
+
188
+ yl.se = yb[22]/16.f;
189
+ yl.sf = yb[23]/4096.f;
190
+
191
+ sumf.s0 += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 0*nb*QK4_0/2, d + ib + 0*nb, sumy, yl, il);
192
+ sumf.s1 += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 1*nb*QK4_0/2, d + ib + 1*nb, sumy, yl, il);
193
+ sumf.s2 += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 2*nb*QK4_0/2, d + ib + 2*nb, sumy, yl, il);
194
+ sumf.s3 += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 3*nb*QK4_0/2, d + ib + 3*nb, sumy, yl, il);
195
+
196
+ sumf.s4 += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 4*nb*QK4_0/2, d + ib + 4*nb, sumy, yl, il);
197
+ sumf.s5 += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 5*nb*QK4_0/2, d + ib + 5*nb, sumy, yl, il);
198
+ sumf.s6 += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 6*nb*QK4_0/2, d + ib + 6*nb, sumy, yl, il);
199
+ sumf.s7 += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 7*nb*QK4_0/2, d + ib + 7*nb, sumy, yl, il);
200
+
201
+ sumf.s8 += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 8*nb*QK4_0/2, d + ib + 8*nb, sumy, yl, il);
202
+ sumf.s9 += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 9*nb*QK4_0/2, d + ib + 9*nb, sumy, yl, il);
203
+ sumf.sa += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 10*nb*QK4_0/2, d + ib + 10*nb, sumy, yl, il);
204
+ sumf.sb += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 11*nb*QK4_0/2, d + ib + 11*nb, sumy, yl, il);
205
+
206
+ sumf.sc += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 12*nb*QK4_0/2, d + ib + 12*nb, sumy, yl, il);
207
+ sumf.sd += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 13*nb*QK4_0/2, d + ib + 13*nb, sumy, yl, il);
208
+ sumf.se += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 14*nb*QK4_0/2, d + ib + 14*nb, sumy, yl, il);
209
+ sumf.sf += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 15*nb*QK4_0/2, d + ib + 15*nb, sumy, yl, il);
210
+
211
+ yb += QK4_0 * (N_SIMDWIDTH/2);
212
+ }
213
+
214
+ float16 tot = (float16)(
215
+ sub_group_reduce_add(sumf.s0), sub_group_reduce_add(sumf.s1),
216
+ sub_group_reduce_add(sumf.s2), sub_group_reduce_add(sumf.s3),
217
+ sub_group_reduce_add(sumf.s4), sub_group_reduce_add(sumf.s5),
218
+ sub_group_reduce_add(sumf.s6), sub_group_reduce_add(sumf.s7),
219
+
220
+ sub_group_reduce_add(sumf.s8), sub_group_reduce_add(sumf.s9),
221
+ sub_group_reduce_add(sumf.sa), sub_group_reduce_add(sumf.sb),
222
+ sub_group_reduce_add(sumf.sc), sub_group_reduce_add(sumf.sd),
223
+ sub_group_reduce_add(sumf.se), sub_group_reduce_add(sumf.sf)
224
+ );
225
+
226
+ if (get_sub_group_local_id() == 0) {
227
+ if (first_row + 0 < ne01) {
228
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 0] = tot.s0;
229
+ }
230
+ if (first_row + 1 < ne01) {
231
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 1] = tot.s1;
232
+ }
233
+ if (first_row + 2 < ne01) {
234
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 2] = tot.s2;
235
+ }
236
+ if (first_row + 3 < ne01) {
237
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 3] = tot.s3;
238
+ }
239
+
240
+ if (first_row + 4 < ne01) {
241
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 4] = tot.s4;
242
+ }
243
+ if (first_row + 5 < ne01) {
244
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 5] = tot.s5;
245
+ }
246
+ if (first_row + 6 < ne01) {
247
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 6] = tot.s6;
248
+ }
249
+ if (first_row + 7 < ne01) {
250
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 7] = tot.s7;
251
+ }
252
+
253
+ if (first_row + 8 < ne01) {
254
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 8] = tot.s8;
255
+ }
256
+ if (first_row + 9 < ne01) {
257
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 9] = tot.s9;
258
+ }
259
+ if (first_row + 10 < ne01) {
260
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 10] = tot.sa;
261
+ }
262
+ if (first_row + 11 < ne01) {
263
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 11] = tot.sb;
264
+ }
265
+
266
+ if (first_row + 12 < ne01) {
267
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 12] = tot.sc;
268
+ }
269
+ if (first_row + 13 < ne01) {
270
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 13] = tot.sd;
271
+ }
272
+ if (first_row + 14 < ne01) {
273
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 14] = tot.se;
274
+ }
275
+ if (first_row + 15 < ne01) {
276
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 15] = tot.sf;
277
+ }
278
+ }
279
+ }
280
+
281
+ #ifdef INTEL_GPU
282
+ REQD_SUBGROUP_SIZE_16
283
+ #elif defined (ADRENO_GPU)
284
+ REQD_SUBGROUP_SIZE_64
285
+ #endif
286
+ kernel void kernel_mul_mat_q4_0_f32_1d_16x_flat(
287
+ global uchar * src0_q,
288
+ global half * src0_d,
289
+ global float * src1,
290
+ ulong offset1,
291
+ global float * dst,
292
+ ulong offsetd,
293
+ int ne00,
294
+ int ne01,
295
+ int ne02,
296
+ int ne10,
297
+ int ne12,
298
+ int ne0,
299
+ int ne1,
300
+ int r2,
301
+ int r3
302
+ ) {
303
+ src1 = (global float*)((global char*)src1 + offset1);
304
+ dst = (global float*)((global char*)dst + offsetd);
305
+
306
+ mul_mat_q_n_f32_1d_16x_flat(src0_q, src0_d, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3);
307
+ }
llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_8x_flat.cl ADDED
@@ -0,0 +1,265 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #pragma OPENCL EXTENSION cl_khr_fp16 : enable
2
+
3
+ #ifdef cl_intel_subgroups
4
+ #pragma OPENCL EXTENSION cl_intel_subgroups : enable
5
+ #else
6
+ #pragma OPENCL EXTENSION cl_khr_subgroups : enable
7
+ #endif
8
+
9
+ #ifdef cl_intel_required_subgroup_size
10
+ #pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
11
+ #define INTEL_GPU 1
12
+ #define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
13
+ #define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
14
+ #elif defined(cl_qcom_reqd_sub_group_size)
15
+ #pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
16
+ #define ADRENO_GPU 1
17
+ #define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
18
+ #define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
19
+ #endif
20
+
21
+ #define QK4_0 32
22
+ #define QR4_0 2
23
+ #define QK4_1 32
24
+ #define QR4_1 2
25
+ #define QK5_0 32
26
+ #define QR5_0 2
27
+ #define QK5_1 32
28
+ #define QR5_1 2
29
+ #define QK8_0 32
30
+ #define QR8_0 1
31
+ #define QK_K 256
32
+ #define K_QUANTS_PER_ITERATION 2
33
+
34
+ typedef char int8_t;
35
+ typedef uchar uint8_t;
36
+ typedef short int16_t;
37
+ typedef ushort uint16_t;
38
+ typedef int int32_t;
39
+ typedef uint uint32_t;
40
+
41
+ //------------------------------------------------------------------------------
42
+ // block_q4_0
43
+ //------------------------------------------------------------------------------
44
+ struct block_q4_0
45
+ {
46
+ half d;
47
+ uint8_t qs[QK4_0 / 2];
48
+ };
49
+
50
+ inline float mm_block_q_4_0_dot_y_flat(
51
+ global uchar * x,
52
+ global half * dh,
53
+ float sumy,
54
+ float16 yl,
55
+ int il
56
+ ) {
57
+ float d = *dh;
58
+ global ushort * qs = ((global ushort *)x + il/2);
59
+ float acc = 0.f;
60
+
61
+ acc += yl.s0 * (qs[0] & 0x000F);
62
+ acc += yl.s1 * (qs[0] & 0x0F00);
63
+ acc += yl.s8 * (qs[0] & 0x00F0);
64
+ acc += yl.s9 * (qs[0] & 0xF000);
65
+
66
+ acc += yl.s2 * (qs[1] & 0x000F);
67
+ acc += yl.s3 * (qs[1] & 0x0F00);
68
+ acc += yl.sa * (qs[1] & 0x00F0);
69
+ acc += yl.sb * (qs[1] & 0xF000);
70
+
71
+ acc += yl.s4 * (qs[2] & 0x000F);
72
+ acc += yl.s5 * (qs[2] & 0x0F00);
73
+ acc += yl.sc * (qs[2] & 0x00F0);
74
+ acc += yl.sd * (qs[2] & 0xF000);
75
+
76
+ acc += yl.s6 * (qs[3] & 0x000F);
77
+ acc += yl.s7 * (qs[3] & 0x0F00);
78
+ acc += yl.se * (qs[3] & 0x00F0);
79
+ acc += yl.sf * (qs[3] & 0xF000);
80
+
81
+ return d * (sumy * -8.f + acc);
82
+ }
83
+
84
+ #ifdef INTEL_GPU
85
+ #define N_DST 8 // each SIMD group works on 8 rows (in weights matrix)
86
+ #define N_SIMDGROUP 1 // number of SIMD groups in a thread group
87
+ #define N_SIMDWIDTH 16 // assuming SIMD group size is 16
88
+ #elif defined (ADRENO_GPU)
89
+ #define N_DST 8
90
+ #define N_SIMDGROUP 1
91
+ #define N_SIMDWIDTH 64
92
+ #endif
93
+ //
94
+ // This variant performs 1d blocking with 8x output.
95
+ // Eeach simdgroup outputs 8 values on `n0` dim (row in the output matrix).
96
+ //
97
+ inline void mul_mat_q_n_f32_1d_8x_flat(
98
+ global uchar * src0_q,
99
+ global half * src0_d,
100
+ global float * src1,
101
+ global float * dst,
102
+ int ne00,
103
+ int ne01,
104
+ int ne02,
105
+ int ne10,
106
+ int ne12,
107
+ int ne0,
108
+ int ne1,
109
+ int r2,
110
+ int r3
111
+ ) {
112
+ const int nb = ne00/QK4_0;
113
+
114
+ int r0 = get_group_id(0);
115
+ int r1 = get_group_id(1);
116
+ int im = get_group_id(2);
117
+
118
+ // (r0 * N_SIMDGROUP + get_sub_group_id()) is the linear global id of
119
+ // a SIMD group in the grid. Each SIMD group produces N_DST values in the
120
+ // result, hence uses nb blocks, i.e., the offset becomes first_row*nb.
121
+ // Currently with llama2 7B, im is always 0.
122
+ // TODO: how to handle im/gqa*(nb*ne0)?
123
+ int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;
124
+
125
+ int i12 = im%ne12;
126
+ int i13 = im/ne12;
127
+
128
+ // The number of scales is the same as the number of blocks.
129
+ ulong offset0_d = first_row * nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);
130
+ // Each block contains QK4_0/2 uchars, hence offset for qs is as follows.
131
+ ulong offset0_q = (first_row * nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02)) * QK4_0/2;
132
+
133
+ global uchar * x = (global uchar *) src0_q + offset0_q;
134
+ global half * d = (global half *) src0_d + offset0_d;
135
+ global float * y = (global float *) src1 + r1*ne10 + im*ne00*ne1;
136
+
137
+ float16 yl;
138
+ float8 sumf = (float8)(0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f);
139
+
140
+ int ix = get_sub_group_local_id()/2;
141
+ int il = 8*(get_sub_group_local_id()%2);
142
+
143
+ global float * yb = y + ix*QK4_0 + il;
144
+
145
+ for (int ib = ix; ib < nb; ib += N_SIMDWIDTH/2) {
146
+ float sumy = 0.f;
147
+
148
+ sumy += yb[0];
149
+ sumy += yb[1];
150
+ sumy += yb[2];
151
+ sumy += yb[3];
152
+ sumy += yb[4];
153
+ sumy += yb[5];
154
+ sumy += yb[6];
155
+ sumy += yb[7];
156
+
157
+ sumy += yb[16];
158
+ sumy += yb[17];
159
+ sumy += yb[18];
160
+ sumy += yb[19];
161
+ sumy += yb[20];
162
+ sumy += yb[21];
163
+ sumy += yb[22];
164
+ sumy += yb[23];
165
+
166
+ yl.s0 = yb[0];
167
+ yl.s1 = yb[1]/256.f;
168
+
169
+ yl.s2 = yb[2];
170
+ yl.s3 = yb[3]/256.f;
171
+
172
+ yl.s4 = yb[4];
173
+ yl.s5 = yb[5]/256.f;
174
+
175
+ yl.s6 = yb[6];
176
+ yl.s7 = yb[7]/256.f;
177
+
178
+ yl.s8 = yb[16]/16.f;
179
+ yl.s9 = yb[17]/4096.f;
180
+
181
+ yl.sa = yb[18]/16.f;
182
+ yl.sb = yb[19]/4096.f;
183
+
184
+ yl.sc = yb[20]/16.f;
185
+ yl.sd = yb[21]/4096.f;
186
+
187
+ yl.se = yb[22]/16.f;
188
+ yl.sf = yb[23]/4096.f;
189
+
190
+ sumf.s0 += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 0*nb*QK4_0/2, d + ib + 0*nb, sumy, yl, il);
191
+ sumf.s1 += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 1*nb*QK4_0/2, d + ib + 1*nb, sumy, yl, il);
192
+ sumf.s2 += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 2*nb*QK4_0/2, d + ib + 2*nb, sumy, yl, il);
193
+ sumf.s3 += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 3*nb*QK4_0/2, d + ib + 3*nb, sumy, yl, il);
194
+
195
+ sumf.s4 += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 4*nb*QK4_0/2, d + ib + 4*nb, sumy, yl, il);
196
+ sumf.s5 += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 5*nb*QK4_0/2, d + ib + 5*nb, sumy, yl, il);
197
+ sumf.s6 += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 6*nb*QK4_0/2, d + ib + 6*nb, sumy, yl, il);
198
+ sumf.s7 += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 7*nb*QK4_0/2, d + ib + 7*nb, sumy, yl, il);
199
+
200
+ yb += QK4_0 * (N_SIMDWIDTH/2);
201
+ }
202
+
203
+ float8 tot = (float8)(
204
+ sub_group_reduce_add(sumf.s0), sub_group_reduce_add(sumf.s1),
205
+ sub_group_reduce_add(sumf.s2), sub_group_reduce_add(sumf.s3),
206
+ sub_group_reduce_add(sumf.s4), sub_group_reduce_add(sumf.s5),
207
+ sub_group_reduce_add(sumf.s6), sub_group_reduce_add(sumf.s7)
208
+ );
209
+
210
+ if (get_sub_group_local_id() == 0) {
211
+ if (first_row + 0 < ne01) {
212
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 0] = tot.s0;
213
+ }
214
+ if (first_row + 1 < ne01) {
215
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 1] = tot.s1;
216
+ }
217
+ if (first_row + 2 < ne01) {
218
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 2] = tot.s2;
219
+ }
220
+ if (first_row + 3 < ne01) {
221
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 3] = tot.s3;
222
+ }
223
+
224
+ if (first_row + 4 < ne01) {
225
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 4] = tot.s4;
226
+ }
227
+ if (first_row + 5 < ne01) {
228
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 5] = tot.s5;
229
+ }
230
+ if (first_row + 6 < ne01) {
231
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 6] = tot.s6;
232
+ }
233
+ if (first_row + 7 < ne01) {
234
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 7] = tot.s7;
235
+ }
236
+ }
237
+ }
238
+
239
+ #ifdef INTEL_GPU
240
+ REQD_SUBGROUP_SIZE_16
241
+ #elif defined (ADRENO_GPU)
242
+ REQD_SUBGROUP_SIZE_64
243
+ #endif
244
+ kernel void kernel_mul_mat_q4_0_f32_1d_8x_flat(
245
+ global uchar * src0_q,
246
+ global half * src0_d,
247
+ global float * src1,
248
+ ulong offset1,
249
+ global float * dst,
250
+ ulong offsetd,
251
+ int ne00,
252
+ int ne01,
253
+ int ne02,
254
+ int ne10,
255
+ int ne12,
256
+ int ne0,
257
+ int ne1,
258
+ int r2,
259
+ int r3
260
+ ) {
261
+ src1 = (global float*)((global char*)src1 + offset1);
262
+ dst = (global float*)((global char*)dst + offsetd);
263
+
264
+ mul_mat_q_n_f32_1d_8x_flat(src0_q, src0_d, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3);
265
+ }
llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_8x_flat.cl ADDED
@@ -0,0 +1,272 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #pragma OPENCL EXTENSION cl_khr_fp16 : enable
2
+
3
+ #ifdef cl_intel_subgroups
4
+ #pragma OPENCL EXTENSION cl_intel_subgroups : enable
5
+ #else
6
+ #pragma OPENCL EXTENSION cl_khr_subgroups : enable
7
+ #endif
8
+
9
+ #ifdef cl_intel_required_subgroup_size
10
+ #pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
11
+ #define INTEL_GPU 1
12
+ #define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
13
+ #define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
14
+ #elif defined(cl_qcom_reqd_sub_group_size)
15
+ #pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
16
+ #define ADRENO_GPU 1
17
+ #define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
18
+ #define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
19
+ #endif
20
+
21
+ #define QK4_0 32
22
+ #define QR4_0 2
23
+ #define QK4_1 32
24
+ #define QR4_1 2
25
+ #define QK5_0 32
26
+ #define QR5_0 2
27
+ #define QK5_1 32
28
+ #define QR5_1 2
29
+ #define QK8_0 32
30
+ #define QR8_0 1
31
+ #define QK_K 256
32
+ #define K_QUANTS_PER_ITERATION 2
33
+
34
+ typedef char int8_t;
35
+ typedef uchar uint8_t;
36
+ typedef short int16_t;
37
+ typedef ushort uint16_t;
38
+ typedef int int32_t;
39
+ typedef uint uint32_t;
40
+
41
+ //------------------------------------------------------------------------------
42
+ // block_q4_0
43
+ //------------------------------------------------------------------------------
44
+ struct block_q4_0
45
+ {
46
+ half d;
47
+ uint8_t qs[QK4_0 / 2];
48
+ };
49
+
50
+ // This function requires the original shuffled weights.
51
+ // As a reminder, the original weights are shuffled so that (q[0], q[16]) are
52
+ // packed together in a byte, so are (q[1], q[17]) and so on.
53
+ inline float block_q_4_0_dot_y_flat(
54
+ global uchar * x,
55
+ global half * dh,
56
+ float sumy,
57
+ float16 yl,
58
+ int il
59
+ ) {
60
+ float d = *dh;
61
+ global ushort * qs = ((global ushort *)x + il/2);
62
+ float acc = 0.f;
63
+
64
+ acc += yl.s0 * (qs[0] & 0x000F);
65
+ acc += yl.s1 * (qs[0] & 0x0F00);
66
+ acc += yl.s8 * (qs[0] & 0x00F0);
67
+ acc += yl.s9 * (qs[0] & 0xF000);
68
+
69
+ acc += yl.s2 * (qs[1] & 0x000F);
70
+ acc += yl.s3 * (qs[1] & 0x0F00);
71
+ acc += yl.sa * (qs[1] & 0x00F0);
72
+ acc += yl.sb * (qs[1] & 0xF000);
73
+
74
+ acc += yl.s4 * (qs[2] & 0x000F);
75
+ acc += yl.s5 * (qs[2] & 0x0F00);
76
+ acc += yl.sc * (qs[2] & 0x00F0);
77
+ acc += yl.sd * (qs[2] & 0xF000);
78
+
79
+ acc += yl.s6 * (qs[3] & 0x000F);
80
+ acc += yl.s7 * (qs[3] & 0x0F00);
81
+ acc += yl.se * (qs[3] & 0x00F0);
82
+ acc += yl.sf * (qs[3] & 0xF000);
83
+
84
+ return d * (sumy * -8.f + acc);
85
+ }
86
+
87
+ //
88
+ // This variant outputs 8 values.
89
+ //
90
+ #undef N_DST
91
+ #undef N_SIMDGROUP
92
+ #undef N_SIMDWIDTH
93
+
94
+ #ifdef INTEL_GPU
95
+ #define N_DST 8 // each SIMD group works on 8 rows
96
+ #define N_SIMDGROUP 1 // number of SIMD groups in a thread group
97
+ #define N_SIMDWIDTH 16 // assuming SIMD group size is 32
98
+ #elif defined (ADRENO_GPU)
99
+ #define N_DST 8
100
+ #define N_SIMDGROUP 1
101
+ #define N_SIMDWIDTH 64
102
+ #endif
103
+
104
+ inline void mul_vec_q_n_f32_8x_flat(
105
+ global uchar * src0_q,
106
+ global half * src0_d,
107
+ global float * src1,
108
+ global float * dst,
109
+ int ne00,
110
+ int ne01,
111
+ int ne02,
112
+ int ne10,
113
+ int ne12,
114
+ int ne0,
115
+ int ne1,
116
+ int r2,
117
+ int r3
118
+ ) {
119
+ const ulong nb = ne00/QK4_0;
120
+
121
+ int r0 = get_group_id(0);
122
+ int r1 = get_group_id(1);
123
+ int im = get_group_id(2);
124
+
125
+ // (r0 * N_SIMDGROUP + get_sub_group_id()) is the linear global id of
126
+ // a SIMD group in the grid. Each SIMD group produces N_DST values in the
127
+ // result, hence uses nb blocks, i.e., the offset becomes first_row*nb.
128
+ // Currently with llama2 7B, im is always 0.
129
+ // TODO: how to handle im/gqa*(nb*ne0)?
130
+ int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;
131
+
132
+ int i12 = im%ne12;
133
+ int i13 = im/ne12;
134
+
135
+ // The number of scales is the same as the number of blocks.
136
+ ulong offset0_d = first_row * nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);
137
+ // Each block contains QK4_0/2 uchars, hence offset for qs is as follows.
138
+ ulong offset0_q = (first_row * nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02)) * QK4_0/2;
139
+
140
+ global uchar * x = (global uchar *) src0_q + offset0_q;
141
+ global half * d = (global half *) src0_d + offset0_d;
142
+ global float * y = (global float *) src1 + r1*ne10 + im*ne00*ne1;
143
+
144
+ float16 yl;
145
+ float8 sumf = 0.f;
146
+
147
+ int ix = get_sub_group_local_id()/2;
148
+ int il = 8*(get_sub_group_local_id()%2);
149
+
150
+ global float * yb = y + ix*QK4_0 + il;
151
+
152
+ for (int ib = ix; ib < nb; ib += N_SIMDWIDTH/2) {
153
+ float sumy = 0.f;
154
+
155
+ sumy += yb[0];
156
+ sumy += yb[1];
157
+ sumy += yb[2];
158
+ sumy += yb[3];
159
+ sumy += yb[4];
160
+ sumy += yb[5];
161
+ sumy += yb[6];
162
+ sumy += yb[7];
163
+
164
+ sumy += yb[16];
165
+ sumy += yb[17];
166
+ sumy += yb[18];
167
+ sumy += yb[19];
168
+ sumy += yb[20];
169
+ sumy += yb[21];
170
+ sumy += yb[22];
171
+ sumy += yb[23];
172
+
173
+ yl.s0 = yb[0];
174
+ yl.s1 = yb[1]/256.f;
175
+
176
+ yl.s2 = yb[2];
177
+ yl.s3 = yb[3]/256.f;
178
+
179
+ yl.s4 = yb[4];
180
+ yl.s5 = yb[5]/256.f;
181
+
182
+ yl.s6 = yb[6];
183
+ yl.s7 = yb[7]/256.f;
184
+
185
+ yl.s8 = yb[16]/16.f;
186
+ yl.s9 = yb[17]/4096.f;
187
+
188
+ yl.sa = yb[18]/16.f;
189
+ yl.sb = yb[19]/4096.f;
190
+
191
+ yl.sc = yb[20]/16.f;
192
+ yl.sd = yb[21]/4096.f;
193
+
194
+ yl.se = yb[22]/16.f;
195
+ yl.sf = yb[23]/4096.f;
196
+
197
+ sumf.s0 += block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 0*nb*QK4_0/2, d + ib + 0*nb, sumy, yl, il);
198
+ sumf.s1 += block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 1*nb*QK4_0/2, d + ib + 1*nb, sumy, yl, il);
199
+ sumf.s2 += block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 2*nb*QK4_0/2, d + ib + 2*nb, sumy, yl, il);
200
+ sumf.s3 += block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 3*nb*QK4_0/2, d + ib + 3*nb, sumy, yl, il);
201
+
202
+ sumf.s4 += block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 4*nb*QK4_0/2, d + ib + 4*nb, sumy, yl, il);
203
+ sumf.s5 += block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 5*nb*QK4_0/2, d + ib + 5*nb, sumy, yl, il);
204
+ sumf.s6 += block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 6*nb*QK4_0/2, d + ib + 6*nb, sumy, yl, il);
205
+ sumf.s7 += block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 7*nb*QK4_0/2, d + ib + 7*nb, sumy, yl, il);
206
+
207
+ yb += QK4_0 * (N_SIMDWIDTH/2);
208
+ }
209
+
210
+ float8 tot = (float8)(
211
+ sub_group_reduce_add(sumf.s0), sub_group_reduce_add(sumf.s1),
212
+ sub_group_reduce_add(sumf.s2), sub_group_reduce_add(sumf.s3),
213
+ sub_group_reduce_add(sumf.s4), sub_group_reduce_add(sumf.s5),
214
+ sub_group_reduce_add(sumf.s6), sub_group_reduce_add(sumf.s7)
215
+ );
216
+
217
+ if (get_sub_group_local_id() == 0) {
218
+ if (first_row + 0 < ne01) {
219
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 0] = tot.s0;
220
+ }
221
+ if (first_row + 1 < ne01) {
222
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 1] = tot.s1;
223
+ }
224
+ if (first_row + 2 < ne01) {
225
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 2] = tot.s2;
226
+ }
227
+ if (first_row + 3 < ne01) {
228
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 3] = tot.s3;
229
+ }
230
+
231
+ if (first_row + 4 < ne01) {
232
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 4] = tot.s4;
233
+ }
234
+ if (first_row + 5 < ne01) {
235
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 5] = tot.s5;
236
+ }
237
+ if (first_row + 6 < ne01) {
238
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 6] = tot.s6;
239
+ }
240
+ if (first_row + 7 < ne01) {
241
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 7] = tot.s7;
242
+ }
243
+ }
244
+ }
245
+
246
+ #ifdef INTEL_GPU
247
+ REQD_SUBGROUP_SIZE_16
248
+ #elif defined (ADRENO_GPU)
249
+ REQD_SUBGROUP_SIZE_64
250
+ #endif
251
+ kernel void kernel_mul_mat_q4_0_f32_8x_flat(
252
+ global uchar * src0_q,
253
+ global half * src0_d,
254
+ global float * src1,
255
+ ulong offset1,
256
+ global float * dst,
257
+ ulong offsetd,
258
+ int ne00,
259
+ int ne01,
260
+ int ne02,
261
+ int ne10,
262
+ int ne12,
263
+ int ne0,
264
+ int ne1,
265
+ int r2,
266
+ int r3
267
+ ) {
268
+ src1 = (global float*)((global char*)src1 + offset1);
269
+ dst = (global float*)((global char*)dst + offsetd);
270
+
271
+ mul_vec_q_n_f32_8x_flat(src0_q, src0_d, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3);
272
+ }
llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_v.cl ADDED
@@ -0,0 +1,254 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #pragma OPENCL EXTENSION cl_khr_fp16 : enable
2
+
3
+ #ifdef cl_intel_subgroups
4
+ #pragma OPENCL EXTENSION cl_intel_subgroups : enable
5
+ #else
6
+ #pragma OPENCL EXTENSION cl_khr_subgroups : enable
7
+ #endif
8
+
9
+ #ifdef cl_intel_required_subgroup_size
10
+ #pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
11
+ #define INTEL_GPU 1
12
+ #define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
13
+ #define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
14
+ #elif defined(cl_qcom_reqd_sub_group_size)
15
+ #pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
16
+ #define ADRENO_GPU 1
17
+ #define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
18
+ #define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
19
+ #endif
20
+
21
+ #define QK4_0 32
22
+ #define QR4_0 2
23
+ #define QK4_1 32
24
+ #define QR4_1 2
25
+ #define QK5_0 32
26
+ #define QR5_0 2
27
+ #define QK5_1 32
28
+ #define QR5_1 2
29
+ #define QK8_0 32
30
+ #define QR8_0 1
31
+ #define QK_K 256
32
+ #define K_QUANTS_PER_ITERATION 2
33
+
34
+ typedef char int8_t;
35
+ typedef uchar uint8_t;
36
+ typedef short int16_t;
37
+ typedef ushort uint16_t;
38
+ typedef int int32_t;
39
+ typedef uint uint32_t;
40
+
41
+ //------------------------------------------------------------------------------
42
+ // block_q4_0
43
+ //------------------------------------------------------------------------------
44
+ struct block_q4_0
45
+ {
46
+ half d;
47
+ uint8_t qs[QK4_0 / 2];
48
+ };
49
+
50
+ //
51
+ // This variant unrolls the loops and uses vector types instead of pointers.
52
+ // It improves performance on Adreno but not so much on Intel.
53
+ //
54
+ inline float block_q_4_0_dot_y_v(
55
+ global struct block_q4_0 * qb_curr,
56
+ float sumy,
57
+ float16 yl,
58
+ int il
59
+ ) {
60
+ float d = qb_curr->d;
61
+ float acc = 0.f;
62
+ global ushort * qs = ((global ushort *)qb_curr + 1 + il/2);
63
+
64
+ acc += yl.s0 * (qs[0] & 0x000F);
65
+ acc += yl.s1 * (qs[0] & 0x0F00);
66
+ acc += yl.s8 * (qs[0] & 0x00F0);
67
+ acc += yl.s9 * (qs[0] & 0xF000);
68
+
69
+ acc += yl.s2 * (qs[1] & 0x000F);
70
+ acc += yl.s3 * (qs[1] & 0x0F00);
71
+ acc += yl.sa * (qs[1] & 0x00F0);
72
+ acc += yl.sb * (qs[1] & 0xF000);
73
+
74
+ acc += yl.s4 * (qs[2] & 0x000F);
75
+ acc += yl.s5 * (qs[2] & 0x0F00);
76
+ acc += yl.sc * (qs[2] & 0x00F0);
77
+ acc += yl.sd * (qs[2] & 0xF000);
78
+
79
+ acc += yl.s6 * (qs[3] & 0x000F);
80
+ acc += yl.s7 * (qs[3] & 0x0F00);
81
+ acc += yl.se * (qs[3] & 0x00F0);
82
+ acc += yl.sf * (qs[3] & 0xF000);
83
+
84
+ return d * (sumy * -8.f + acc);
85
+ }
86
+
87
+ #undef N_DST
88
+ #undef N_SIMDGROUP
89
+ #undef N_SIMDWIDTH
90
+
91
+ #ifdef INTEL_GPU
92
+ #define N_DST 4 // each SIMD group works on 4 rows
93
+ #define N_SIMDGROUP 1 // number of SIMD groups in a thread group
94
+ #define N_SIMDWIDTH 16 // assuming SIMD group size is 16
95
+ #elif defined (ADRENO_GPU)
96
+ #define N_DST 4
97
+ #define N_SIMDGROUP 1
98
+ #define N_SIMDWIDTH 64
99
+ #endif
100
+
101
+ inline void mul_vec_q_n_f32_v(
102
+ global void * src0,
103
+ global float * src1,
104
+ global float * dst,
105
+ int ne00,
106
+ int ne01,
107
+ int ne02,
108
+ int ne10,
109
+ int ne12,
110
+ int ne0,
111
+ int ne1,
112
+ int r2,
113
+ int r3
114
+ ) {
115
+ const ulong nb = ne00/QK4_0;
116
+
117
+ int r0 = get_group_id(0);
118
+ int r1 = get_group_id(1);
119
+ int im = get_group_id(2);
120
+
121
+ // (r0 * N_SIMDGROUP + get_sub_group_id()) is essenatially the linear global
122
+ // id of a SIMD group in the grid.
123
+ int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;
124
+
125
+ int i12 = im%ne12;
126
+ int i13 = im/ne12;
127
+
128
+ ulong offset0 = first_row * nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);
129
+
130
+ global struct block_q4_0 * x = (global struct block_q4_0 *) src0 + offset0;
131
+ global float * y = (global float *) src1 + r1*ne10 + im*ne00*ne1;
132
+
133
+ float16 yl; // src1 vector cache
134
+ float4 sumf = (float4)(0.f, 0.f, 0.f, 0.f);
135
+
136
+ int ix = get_sub_group_local_id()/2;
137
+ int il = 8*(get_sub_group_local_id()%2);
138
+
139
+ global float * yb = y + ix * QK4_0 + il;
140
+
141
+ // each thread in a SIMD group deals with half a block.
142
+ for (int ib = ix; ib < nb; ib += N_SIMDWIDTH/2) {
143
+ float sumy = 0;
144
+
145
+ sumy += yb[0];
146
+ sumy += yb[1];
147
+ sumy += yb[2];
148
+ sumy += yb[3];
149
+ sumy += yb[4];
150
+ sumy += yb[5];
151
+ sumy += yb[6];
152
+ sumy += yb[7];
153
+
154
+ sumy += yb[16];
155
+ sumy += yb[17];
156
+ sumy += yb[18];
157
+ sumy += yb[19];
158
+ sumy += yb[20];
159
+ sumy += yb[21];
160
+ sumy += yb[22];
161
+ sumy += yb[23];
162
+
163
+
164
+ yl.s0 = yb[0];
165
+ yl.s1 = yb[1]/256.f;
166
+
167
+ yl.s2 = yb[2];
168
+ yl.s3 = yb[3]/256.f;
169
+
170
+ yl.s4 = yb[4];
171
+ yl.s5 = yb[5]/256.f;
172
+
173
+ yl.s6 = yb[6];
174
+ yl.s7 = yb[7]/256.f;
175
+
176
+ yl.s8 = yb[16]/16.f;
177
+ yl.s9 = yb[17]/4096.f;
178
+
179
+ yl.sa = yb[18]/16.f;
180
+ yl.sb = yb[19]/4096.f;
181
+
182
+ yl.sc = yb[20]/16.f;
183
+ yl.sd = yb[21]/4096.f;
184
+
185
+ yl.se = yb[22]/16.f;
186
+ yl.sf = yb[23]/4096.f;
187
+
188
+ sumf.s0 += block_q_4_0_dot_y_v(x+ib+0*nb, sumy, yl, il);
189
+ sumf.s1 += block_q_4_0_dot_y_v(x+ib+1*nb, sumy, yl, il);
190
+ sumf.s2 += block_q_4_0_dot_y_v(x+ib+2*nb, sumy, yl, il);
191
+ sumf.s3 += block_q_4_0_dot_y_v(x+ib+3*nb, sumy, yl, il);
192
+
193
+ // One thread in a SIMD group (i.e., subgroup) handles a half block,
194
+ // hence then entire SIMD group handles SIMDWIDTH/2 blocks.
195
+ // y points to the activation matrix (of type float). Therefore for
196
+ // one thread, the # of blocks y should advance is SIMDWIDTH/2 (because
197
+ // SIMDWIDTH/2 blocks are processed by a SIMD group) - in terms of
198
+ // floats, it is QK4_0 * (SIMDWIDTH/2), where QK4_0 is the block size.
199
+ yb += QK4_0 * (N_SIMDWIDTH/2);
200
+ }
201
+
202
+ // The above does not work for Adreno - it produces incorrect results for
203
+ // row = 1, 2, 3 and only row = 0 gives the correct result.
204
+ // If N_DST is changed, the below array must be initialized accordingly.
205
+ // This also seems to perform better on Intel.
206
+ float4 tot = (float4)(
207
+ sub_group_reduce_add(sumf.s0), sub_group_reduce_add(sumf.s1),
208
+ sub_group_reduce_add(sumf.s2), sub_group_reduce_add(sumf.s3)
209
+ );
210
+
211
+ if (get_sub_group_local_id() == 0) {
212
+ if (first_row + 0 < ne01) {
213
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 0] = tot.s0;
214
+ }
215
+ if (first_row + 1 < ne01) {
216
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 1] = tot.s1;
217
+ }
218
+ if (first_row + 2 < ne01) {
219
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 2] = tot.s2;
220
+ }
221
+ if (first_row + 3 < ne01) {
222
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 3] = tot.s3;
223
+ }
224
+ }
225
+ }
226
+
227
+ #ifdef INTEL_GPU
228
+ REQD_SUBGROUP_SIZE_16
229
+ #elif defined (ADRENO_GPU)
230
+ REQD_SUBGROUP_SIZE_64
231
+ #endif
232
+ kernel void kernel_mul_mat_q4_0_f32_v(
233
+ global void * src0,
234
+ ulong offset0,
235
+ global float * src1,
236
+ ulong offset1,
237
+ global float * dst,
238
+ ulong offsetd,
239
+ int ne00,
240
+ int ne01,
241
+ int ne02,
242
+ int ne10,
243
+ int ne12,
244
+ int ne0,
245
+ int ne1,
246
+ int r2,
247
+ int r3
248
+ ) {
249
+ src0 = (global void*)((global char*)src0 + offset0);
250
+ src1 = (global float*)((global char*)src1 + offset1);
251
+ dst = (global float*)((global char*)dst + offsetd);
252
+
253
+ mul_vec_q_n_f32_v(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3);
254
+ }
llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_q4_1_f32.cl ADDED
@@ -0,0 +1,219 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #pragma OPENCL EXTENSION cl_khr_fp16 : enable
2
+
3
+ #ifdef cl_intel_subgroups
4
+ #pragma OPENCL EXTENSION cl_intel_subgroups : enable
5
+ #else
6
+ #pragma OPENCL EXTENSION cl_khr_subgroups : enable
7
+ #endif
8
+
9
+ #ifdef cl_intel_required_subgroup_size
10
+ #pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
11
+ #define INTEL_GPU 1
12
+ #define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
13
+ #define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
14
+ #elif defined(cl_qcom_reqd_sub_group_size)
15
+ #pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
16
+ #define ADRENO_GPU 1
17
+ #define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
18
+ #define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
19
+ #endif
20
+
21
+ #define QK4_1 32
22
+
23
+ struct block_q4_1 {
24
+ half d; // delta
25
+ half m; // min
26
+ uchar qs[QK4_1 / 2]; // nibbles / quants
27
+ };
28
+
29
+ inline float block_q4_1_dot_y(
30
+ global const struct block_q4_1 * qb_curr,
31
+ float sumy,
32
+ float16 yl,
33
+ int il
34
+ ) {
35
+ float d = qb_curr->d;
36
+ float m = qb_curr->m;
37
+
38
+ float4 acc = (float4)(0.0f, 0.0f, 0.0f, 0.0f);
39
+
40
+ global const ushort * qs = ((global const ushort *) qb_curr + 2 + il/2);
41
+
42
+ acc.s0 += yl.s0 * (qs[0] & 0x000F);
43
+ acc.s0 += yl.s1 * (qs[0] & 0x0F00);
44
+ acc.s0 += yl.s8 * (qs[0] & 0x00F0);
45
+ acc.s3 += yl.s9 * (qs[0] & 0xF000);
46
+
47
+ acc.s0 += yl.s2 * (qs[1] & 0x000F);
48
+ acc.s1 += yl.s3 * (qs[1] & 0x0F00);
49
+ acc.s2 += yl.sa * (qs[1] & 0x00F0);
50
+ acc.s3 += yl.sb * (qs[1] & 0xF000);
51
+
52
+ acc.s0 += yl.s4 * (qs[2] & 0x000F);
53
+ acc.s1 += yl.s5 * (qs[2] & 0x0F00);
54
+ acc.s2 += yl.sc * (qs[2] & 0x00F0);
55
+ acc.s3 += yl.sd * (qs[2] & 0xF000);
56
+
57
+ acc.s0 += yl.s6 * (qs[3] & 0x000F);
58
+ acc.s1 += yl.s7 * (qs[3] & 0x0F00);
59
+ acc.s2 += yl.se * (qs[3] & 0x00F0);
60
+ acc.s3 += yl.sf * (qs[3] & 0xF000);
61
+
62
+ return d * (acc.s0 + acc.s1 + acc.s2 + acc.s3) + sumy * m;
63
+ }
64
+
65
+ #undef N_DST
66
+ #undef N_SIMDGROUP
67
+ #undef N_SIMDWIDTH
68
+
69
+ #ifdef INTEL_GPU
70
+ #define N_DST 4 // each subgroup works on 4 rows
71
+ #define N_SIMDGROUP 1 // number of subgroups in a thread group
72
+ #define N_SIMDWIDTH 16 // assuming subgroup size is 16
73
+ #elif defined (ADRENO_GPU)
74
+ #define N_DST 4
75
+ #define N_SIMDGROUP 1
76
+ #define N_SIMDWIDTH 64
77
+ #endif
78
+
79
+ inline void mul_vec_q_n_f32(
80
+ global void * src0,
81
+ global float * src1,
82
+ global float * dst,
83
+ int ne00,
84
+ int ne01,
85
+ int ne02,
86
+ int ne10,
87
+ int ne12,
88
+ int ne0,
89
+ int ne1,
90
+ int r2,
91
+ int r3
92
+ ) {
93
+ const ulong nb = ne00/QK4_1;
94
+
95
+ int r0 = get_group_id(0);
96
+ int r1 = get_group_id(1);
97
+ int im = get_group_id(2);
98
+
99
+ int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;
100
+
101
+ int i12 = im%ne12;
102
+ int i13 = im/ne12;
103
+
104
+ ulong offset0 = first_row * nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);
105
+
106
+ global struct block_q4_1 * x = (global struct block_q4_1 *) src0 + offset0;
107
+ global float * y = (global float *) src1 + r1*ne10 + im*ne00*ne1;
108
+
109
+ float16 yl;
110
+ float4 sumf = (float4)(0.f, 0.f, 0.f, 0.f);
111
+
112
+ int ix = get_sub_group_local_id()/2;
113
+ int il = 8*(get_sub_group_local_id()%2);
114
+
115
+ global float * yb = y + ix * QK4_1 + il;
116
+
117
+ for (int ib = ix; ib < nb; ib += N_SIMDWIDTH/2) {
118
+ float sumy = 0;
119
+
120
+ sumy += yb[0];
121
+ sumy += yb[1];
122
+ sumy += yb[2];
123
+ sumy += yb[3];
124
+ sumy += yb[4];
125
+ sumy += yb[5];
126
+ sumy += yb[6];
127
+ sumy += yb[7];
128
+
129
+ sumy += yb[16];
130
+ sumy += yb[17];
131
+ sumy += yb[18];
132
+ sumy += yb[19];
133
+ sumy += yb[20];
134
+ sumy += yb[21];
135
+ sumy += yb[22];
136
+ sumy += yb[23];
137
+
138
+
139
+ yl.s0 = yb[0];
140
+ yl.s1 = yb[1]/256.f;
141
+
142
+ yl.s2 = yb[2];
143
+ yl.s3 = yb[3]/256.f;
144
+
145
+ yl.s4 = yb[4];
146
+ yl.s5 = yb[5]/256.f;
147
+
148
+ yl.s6 = yb[6];
149
+ yl.s7 = yb[7]/256.f;
150
+
151
+ yl.s8 = yb[16]/16.f;
152
+ yl.s9 = yb[17]/4096.f;
153
+
154
+ yl.sa = yb[18]/16.f;
155
+ yl.sb = yb[19]/4096.f;
156
+
157
+ yl.sc = yb[20]/16.f;
158
+ yl.sd = yb[21]/4096.f;
159
+
160
+ yl.se = yb[22]/16.f;
161
+ yl.sf = yb[23]/4096.f;
162
+
163
+ sumf.s0 += block_q4_1_dot_y(x+ib+0*nb, sumy, yl, il);
164
+ sumf.s1 += block_q4_1_dot_y(x+ib+1*nb, sumy, yl, il);
165
+ sumf.s2 += block_q4_1_dot_y(x+ib+2*nb, sumy, yl, il);
166
+ sumf.s3 += block_q4_1_dot_y(x+ib+3*nb, sumy, yl, il);
167
+
168
+ yb += QK4_1 * (N_SIMDWIDTH/2);
169
+ }
170
+
171
+ float4 tot = (float4)(
172
+ sub_group_reduce_add(sumf.s0), sub_group_reduce_add(sumf.s1),
173
+ sub_group_reduce_add(sumf.s2), sub_group_reduce_add(sumf.s3)
174
+ );
175
+
176
+ if (get_sub_group_local_id() == 0) {
177
+ if (first_row + 0 < ne01) {
178
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 0] = tot.s0;
179
+ }
180
+ if (first_row + 1 < ne01) {
181
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 1] = tot.s1;
182
+ }
183
+ if (first_row + 2 < ne01) {
184
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 2] = tot.s2;
185
+ }
186
+ if (first_row + 3 < ne01) {
187
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 3] = tot.s3;
188
+ }
189
+ }
190
+ }
191
+
192
+ #ifdef INTEL_GPU
193
+ REQD_SUBGROUP_SIZE_16
194
+ #elif defined (ADRENO_GPU)
195
+ REQD_SUBGROUP_SIZE_64
196
+ #endif
197
+ kernel void kernel_mul_mv_q4_1_f32(
198
+ global void * src0,
199
+ ulong offset0,
200
+ global float * src1,
201
+ ulong offset1,
202
+ global float * dst,
203
+ ulong offsetd,
204
+ int ne00,
205
+ int ne01,
206
+ int ne02,
207
+ int ne10,
208
+ int ne12,
209
+ int ne0,
210
+ int ne1,
211
+ int r2,
212
+ int r3
213
+ ) {
214
+ src0 = (global void*)((global char*)src0 + offset0);
215
+ src1 = (global float*)((global char*)src1 + offset1);
216
+ dst = (global float*)((global char*)dst + offsetd);
217
+
218
+ mul_vec_q_n_f32(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3);
219
+ }
llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_q4_1_f32_flat.cl ADDED
@@ -0,0 +1,229 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #pragma OPENCL EXTENSION cl_khr_fp16 : enable
2
+
3
+ #ifdef cl_intel_subgroups
4
+ #pragma OPENCL EXTENSION cl_intel_subgroups : enable
5
+ #else
6
+ #pragma OPENCL EXTENSION cl_khr_subgroups : enable
7
+ #endif
8
+
9
+ #ifdef cl_intel_required_subgroup_size
10
+ #pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
11
+ #define INTEL_GPU 1
12
+ #define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
13
+ #define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
14
+ #elif defined(cl_qcom_reqd_sub_group_size)
15
+ #pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
16
+ #define ADRENO_GPU 1
17
+ #define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
18
+ #define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
19
+ #endif
20
+
21
+ #define QK4_1 32
22
+
23
+ struct block_q4_1 {
24
+ half d; // delta
25
+ half m; // min
26
+ uchar qs[QK4_1 / 2]; // nibbles / quants
27
+ };
28
+
29
+ inline float block_q4_1_dot_y_flat(
30
+ global const uchar * x,
31
+ global const half * dh,
32
+ global const half * mh,
33
+ float sumy,
34
+ float16 yl,
35
+ int il
36
+ ) {
37
+ float d = *dh;
38
+ float m = *mh;
39
+ global const ushort * qs = ((global const ushort *) x + il/2);
40
+
41
+ float4 acc = (float4)(0.0f, 0.0f, 0.0f, 0.0f);
42
+
43
+ acc.s0 += yl.s0 * (qs[0] & 0x000F);
44
+ acc.s0 += yl.s1 * (qs[0] & 0x0F00);
45
+ acc.s0 += yl.s8 * (qs[0] & 0x00F0);
46
+ acc.s3 += yl.s9 * (qs[0] & 0xF000);
47
+
48
+ acc.s0 += yl.s2 * (qs[1] & 0x000F);
49
+ acc.s1 += yl.s3 * (qs[1] & 0x0F00);
50
+ acc.s2 += yl.sa * (qs[1] & 0x00F0);
51
+ acc.s3 += yl.sb * (qs[1] & 0xF000);
52
+
53
+ acc.s0 += yl.s4 * (qs[2] & 0x000F);
54
+ acc.s1 += yl.s5 * (qs[2] & 0x0F00);
55
+ acc.s2 += yl.sc * (qs[2] & 0x00F0);
56
+ acc.s3 += yl.sd * (qs[2] & 0xF000);
57
+
58
+ acc.s0 += yl.s6 * (qs[3] & 0x000F);
59
+ acc.s1 += yl.s7 * (qs[3] & 0x0F00);
60
+ acc.s2 += yl.se * (qs[3] & 0x00F0);
61
+ acc.s3 += yl.sf * (qs[3] & 0xF000);
62
+
63
+ return d * (acc.s0 + acc.s1 + acc.s2 + acc.s3) + sumy * m;
64
+ }
65
+
66
+ #undef N_DST
67
+ #undef N_SIMDGROUP
68
+ #undef N_SIMDWIDTH
69
+
70
+ #ifdef INTEL_GPU
71
+ #define N_DST 4 // each subgroup works on 4 rows
72
+ #define N_SIMDGROUP 1 // number of subgroups in a thread group
73
+ #define N_SIMDWIDTH 16 // assuming subgroup size is 16
74
+ #elif defined (ADRENO_GPU)
75
+ #define N_DST 4
76
+ #define N_SIMDGROUP 1
77
+ #define N_SIMDWIDTH 64
78
+ #endif
79
+
80
+ inline void mul_vec_q_n_f32_flat(
81
+ global void * src0_q,
82
+ global void * src0_d,
83
+ global void * src0_m,
84
+ global float * src1,
85
+ global float * dst,
86
+ int ne00,
87
+ int ne01,
88
+ int ne02,
89
+ int ne10,
90
+ int ne12,
91
+ int ne0,
92
+ int ne1,
93
+ int r2,
94
+ int r3
95
+ ) {
96
+ const ulong nb = ne00/QK4_1;
97
+
98
+ int r0 = get_group_id(0);
99
+ int r1 = get_group_id(1);
100
+ int im = get_group_id(2);
101
+
102
+ int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;
103
+
104
+ int i12 = im%ne12;
105
+ int i13 = im/ne12;
106
+
107
+ ulong offset0 = first_row * nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);
108
+
109
+ // The number of scales/mins is the same as the number of blocks.
110
+ ulong offset0_dm = (first_row * nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02));
111
+ // Each block contains QK4_1/2 uchars, hence offset for qs is as follows.
112
+ ulong offset0_q = (first_row * nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02)) * QK4_1/2;
113
+
114
+ global uchar * x = (global uchar *) src0_q + offset0_q;
115
+ global half * d = (global half *) src0_d + offset0_dm;
116
+ global half * m = (global half *) src0_m + offset0_dm;
117
+ global float * y = (global float *) src1 + r1*ne10 + im*ne00*ne1;
118
+
119
+ float16 yl;
120
+ float4 sumf = (float4)(0.f, 0.f, 0.f, 0.f);
121
+
122
+ int ix = get_sub_group_local_id()/2;
123
+ int il = 8*(get_sub_group_local_id()%2);
124
+
125
+ global float * yb = y + ix * QK4_1 + il;
126
+
127
+ for (int ib = ix; ib < nb; ib += N_SIMDWIDTH/2) {
128
+ float sumy = 0;
129
+
130
+ sumy += yb[0];
131
+ sumy += yb[1];
132
+ sumy += yb[2];
133
+ sumy += yb[3];
134
+ sumy += yb[4];
135
+ sumy += yb[5];
136
+ sumy += yb[6];
137
+ sumy += yb[7];
138
+
139
+ sumy += yb[16];
140
+ sumy += yb[17];
141
+ sumy += yb[18];
142
+ sumy += yb[19];
143
+ sumy += yb[20];
144
+ sumy += yb[21];
145
+ sumy += yb[22];
146
+ sumy += yb[23];
147
+
148
+
149
+ yl.s0 = yb[0];
150
+ yl.s1 = yb[1]/256.f;
151
+
152
+ yl.s2 = yb[2];
153
+ yl.s3 = yb[3]/256.f;
154
+
155
+ yl.s4 = yb[4];
156
+ yl.s5 = yb[5]/256.f;
157
+
158
+ yl.s6 = yb[6];
159
+ yl.s7 = yb[7]/256.f;
160
+
161
+ yl.s8 = yb[16]/16.f;
162
+ yl.s9 = yb[17]/4096.f;
163
+
164
+ yl.sa = yb[18]/16.f;
165
+ yl.sb = yb[19]/4096.f;
166
+
167
+ yl.sc = yb[20]/16.f;
168
+ yl.sd = yb[21]/4096.f;
169
+
170
+ yl.se = yb[22]/16.f;
171
+ yl.sf = yb[23]/4096.f;
172
+
173
+ sumf.s0 += block_q4_1_dot_y_flat(x + ib*QK4_1/2 + 0*nb*QK4_1/2, d + ib + 0*nb, m + ib + 0*nb, sumy, yl, il);
174
+ sumf.s1 += block_q4_1_dot_y_flat(x + ib*QK4_1/2 + 1*nb*QK4_1/2, d + ib + 1*nb, m + ib + 1*nb, sumy, yl, il);
175
+ sumf.s2 += block_q4_1_dot_y_flat(x + ib*QK4_1/2 + 2*nb*QK4_1/2, d + ib + 2*nb, m + ib + 2*nb, sumy, yl, il);
176
+ sumf.s3 += block_q4_1_dot_y_flat(x + ib*QK4_1/2 + 3*nb*QK4_1/2, d + ib + 3*nb, m + ib + 3*nb, sumy, yl, il);
177
+
178
+ yb += QK4_1 * (N_SIMDWIDTH/2);
179
+ }
180
+
181
+ float4 tot = (float4)(
182
+ sub_group_reduce_add(sumf.s0), sub_group_reduce_add(sumf.s1),
183
+ sub_group_reduce_add(sumf.s2), sub_group_reduce_add(sumf.s3)
184
+ );
185
+
186
+ if (get_sub_group_local_id() == 0) {
187
+ if (first_row + 0 < ne01) {
188
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 0] = tot.s0;
189
+ }
190
+ if (first_row + 1 < ne01) {
191
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 1] = tot.s1;
192
+ }
193
+ if (first_row + 2 < ne01) {
194
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 2] = tot.s2;
195
+ }
196
+ if (first_row + 3 < ne01) {
197
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 3] = tot.s3;
198
+ }
199
+ }
200
+ }
201
+
202
+ #ifdef INTEL_GPU
203
+ REQD_SUBGROUP_SIZE_16
204
+ #elif defined (ADRENO_GPU)
205
+ REQD_SUBGROUP_SIZE_64
206
+ #endif
207
+ kernel void kernel_mul_mv_q4_1_f32_flat(
208
+ global void * src0_q,
209
+ global void * src0_d,
210
+ global void * src0_m,
211
+ global float * src1,
212
+ ulong offset1,
213
+ global float * dst,
214
+ ulong offsetd,
215
+ int ne00,
216
+ int ne01,
217
+ int ne02,
218
+ int ne10,
219
+ int ne12,
220
+ int ne0,
221
+ int ne1,
222
+ int r2,
223
+ int r3
224
+ ) {
225
+ src1 = (global float*)((global char*)src1 + offset1);
226
+ dst = (global float*)((global char*)dst + offsetd);
227
+
228
+ mul_vec_q_n_f32_flat(src0_q, src0_d, src0_m, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3);
229
+ }
llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32.cl ADDED
@@ -0,0 +1,180 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #ifdef cl_intel_required_subgroup_size
2
+ #pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
3
+ #define INTEL_GPU 1
4
+ #define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
5
+ #define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
6
+ #elif defined(cl_qcom_reqd_sub_group_size)
7
+ #pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
8
+ #define ADRENO_GPU 1
9
+ #define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
10
+ #define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
11
+ #endif
12
+
13
+ //------------------------------------------------------------------------------
14
+ // block_q4_K
15
+ //------------------------------------------------------------------------------
16
+ #define QK_K 256
17
+ #define K_SCALE_SIZE 12
18
+
19
+ // 8 blocks of 32 elements each
20
+ // weight is represented as x = a * q + b
21
+ typedef struct {
22
+ half d; // super-block scale for quantized scales
23
+ half dmin; // super-block scale for quantized mins
24
+
25
+ uchar scales[K_SCALE_SIZE]; // scales and mins, quantized with 6 bits
26
+ uchar qs[QK_K/2]; // 4-bit quants
27
+ } block_q4_K;
28
+
29
+ #undef N_DST
30
+ #undef N_SIMDGROUP
31
+ #undef N_SIMDWIDTH
32
+
33
+ #ifdef INTEL_GPU
34
+ #define N_DST 4 // number of rows each SIMD group works on
35
+ #define N_SIMDGROUP 1 // number of SIMD groups in a thread group
36
+ #define N_SIMDWIDTH 16 // SIMD group size
37
+ #elif defined (ADRENO_GPU)
38
+ #define N_DST 4
39
+ #define N_SIMDGROUP 1
40
+ #define N_SIMDWIDTH 64
41
+ #endif
42
+
43
+ #undef BLOCK_STRIDE
44
+ // number of (super) blocks each subgroup processes
45
+ // each thread in a subgroup processes a block (32 weights)
46
+ #define BLOCK_STRIDE (N_SIMDWIDTH/8)
47
+
48
+ #ifdef INTEL_GPU
49
+ REQD_SUBGROUP_SIZE_16
50
+ #elif defined (ADRENO_GPU)
51
+ REQD_SUBGROUP_SIZE_64
52
+ #endif
53
+ kernel void kernel_mul_mv_q4_K_f32(
54
+ global char * src0,
55
+ int offset0,
56
+ global char * src1,
57
+ int offset1,
58
+ global char * dst,
59
+ int offsetd,
60
+ int ne00,
61
+ int ne01,
62
+ ulong nb01,
63
+ ulong nb02,
64
+ ulong nb03,
65
+ int ne12,
66
+ ulong nb11,
67
+ ulong nb12,
68
+ ulong nb13,
69
+ int ne0,
70
+ int ne1,
71
+ int r2,
72
+ int r3
73
+ ) {
74
+ src0 = src0 + offset0;
75
+ src1 = src1 + offset1;
76
+ dst = dst + offsetd;
77
+
78
+ ushort kmask1 = 0x3f3f;
79
+ ushort kmask2 = 0x0f0f;
80
+ ushort kmask3 = 0xc0c0;
81
+
82
+ int ix = get_sub_group_local_id()/8; // super block index
83
+ int it = get_sub_group_local_id()%8; // block index (inside super block)
84
+ int iq = it/4; // 0 or 1 - first or second half of the super block
85
+ int ir = it%4; // 0...3 - block index in the half super block
86
+
87
+ int nb = ne00/QK_K;
88
+
89
+ int r0 = get_group_id(0);
90
+ int r1 = get_group_id(1);
91
+ int im = get_group_id(2);
92
+ int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;
93
+
94
+ int i12 = im%ne12;
95
+ int i13 = im/ne12;
96
+
97
+ int offset_src0 = first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03;
98
+ int offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13;
99
+
100
+ global block_q4_K * x = (global block_q4_K *) (src0 + offset_src0);
101
+ global float * y = (global float *) (src1 + offset_src1);
102
+
103
+ float yl[16];
104
+ float yh[16];
105
+ float sumf[N_DST] = {0.f};
106
+ float all_sum;
107
+
108
+ global float * y4 = y + ix * QK_K + 64 * iq + 8 * ir;
109
+
110
+ ushort sc16[4];
111
+ uchar * sc8 = (uchar *)sc16;
112
+
113
+ for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) {
114
+ float4 sumy = {0.f, 0.f, 0.f, 0.f};
115
+ for (int i = 0; i < 8; ++i) {
116
+ yl[i+0] = y4[i+0];
117
+ sumy.s0 += yl[i+0];
118
+
119
+ yl[i+8] = y4[i+32];
120
+ sumy.s1 += yl[i+8];
121
+
122
+ yh[i+0] = y4[i+128];
123
+ sumy.s2 += yh[i+0];
124
+
125
+ yh[i+8] = y4[i+160];
126
+ sumy.s3 += yh[i+8];
127
+ }
128
+
129
+ global ushort * sc = (global ushort *)x[ib].scales + iq;
130
+ global ushort * q1 = (global ushort *)x[ib].qs + 16 * iq + 4 * ir;
131
+ global half * dh = &x[ib].d;
132
+
133
+ for (int row = 0; row < N_DST; row++) {
134
+ sc16[0] = sc[0] & kmask1;
135
+ sc16[1] = sc[2] & kmask1;
136
+ sc16[2] = ((sc[4] >> 0) & kmask2) | ((sc[0] & kmask3) >> 2);
137
+ sc16[3] = ((sc[4] >> 4) & kmask2) | ((sc[2] & kmask3) >> 2);
138
+
139
+ global ushort * q2 = q1 + 32;
140
+
141
+ float4 acc1 = {0.f, 0.f, 0.f, 0.f};
142
+ float4 acc2 = {0.f, 0.f, 0.f, 0.f};
143
+ for (int i = 0; i < 8; i += 2) {
144
+ acc1.s0 += yl[i+0] * (q1[i/2] & 0x000F);
145
+ acc1.s1 += yl[i+1] * (q1[i/2] & 0x0F00);
146
+ acc1.s2 += yl[i+8] * (q1[i/2] & 0x00F0);
147
+ acc1.s3 += yl[i+9] * (q1[i/2] & 0xF000);
148
+ acc2.s0 += yh[i+0] * (q2[i/2] & 0x000F);
149
+ acc2.s1 += yh[i+1] * (q2[i/2] & 0x0F00);
150
+ acc2.s2 += yh[i+8] * (q2[i/2] & 0x00F0);
151
+ acc2.s3 += yh[i+9] * (q2[i/2] & 0xF000);
152
+ }
153
+
154
+ float dall = dh[0];
155
+ float dmin = dh[1];
156
+ sumf[row] += dall * ((acc1.s0 + 1.f/256.f * acc1.s1) * sc8[0] +
157
+ (acc1.s2 + 1.f/256.f * acc1.s3) * sc8[1] * 1.f/16.f +
158
+ (acc2.s0 + 1.f/256.f * acc2.s1) * sc8[4] +
159
+ (acc2.s2 + 1.f/256.f * acc2.s3) * sc8[5] * 1.f/16.f) -
160
+ dmin * (sumy.s0 * sc8[2] + sumy.s1 * sc8[3] + sumy.s2 * sc8[6] + sumy.s3 * sc8[7]);
161
+
162
+ q1 += nb01/2;
163
+ sc += nb01/2;
164
+ dh += nb01/2;
165
+ }
166
+
167
+ y4 += BLOCK_STRIDE * QK_K;
168
+ }
169
+
170
+ global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0;
171
+
172
+ for (int row = 0; row < N_DST; ++row) {
173
+ all_sum = sub_group_reduce_add(sumf[row]);
174
+ if (first_row + row < ne01) {
175
+ if (get_sub_group_local_id() == 0) {
176
+ dst_f32[first_row + row] = all_sum;
177
+ }
178
+ }
179
+ }
180
+ }
llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32_flat.cl ADDED
@@ -0,0 +1,196 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #pragma OPENCL EXTENSION cl_khr_fp16 : enable
2
+
3
+ #ifdef cl_intel_subgroups
4
+ #pragma OPENCL EXTENSION cl_intel_subgroups : enable
5
+ #else
6
+ #pragma OPENCL EXTENSION cl_khr_subgroups : enable
7
+ #endif
8
+
9
+ #ifdef cl_intel_required_subgroup_size
10
+ #pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
11
+ #define INTEL_GPU 1
12
+ #define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
13
+ #define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
14
+ #elif defined(cl_qcom_reqd_sub_group_size)
15
+ #pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
16
+ #define ADRENO_GPU 1
17
+ #define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
18
+ #define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
19
+ #endif
20
+
21
+ //------------------------------------------------------------------------------
22
+ // block_q4_K
23
+ //------------------------------------------------------------------------------
24
+ #define QK_K 256
25
+ #define BLOCK_Q4K_SIZE 144
26
+ #define K_SCALE_SIZE 12
27
+
28
+ // 8 blocks of 32 elements each
29
+ // weight is represented as x = a * q + b
30
+ typedef struct {
31
+ half d; // super-block scale for quantized scales
32
+ half dmin; // super-block scale for quantized mins
33
+
34
+ uchar scales[K_SCALE_SIZE]; // scales and mins, quantized with 6 bits
35
+ uchar qs[QK_K/2]; // 4-bit quants
36
+ } block_q4_K;
37
+
38
+ #undef N_DST
39
+ #undef N_SIMDGROUP
40
+ #undef N_SIMDWIDTH
41
+
42
+ #ifdef INTEL_GPU
43
+ #define N_DST 4 // number of rows each SIMD group works on
44
+ #define N_SIMDGROUP 1 // number of SIMD groups in a thread group
45
+ #define N_SIMDWIDTH 16 // SIMD group size
46
+ #elif defined (ADRENO_GPU)
47
+ #define N_DST 16
48
+ #define N_SIMDGROUP 2
49
+ #define N_SIMDWIDTH 64
50
+ #endif
51
+
52
+ #undef BLOCK_STRIDE
53
+ // number of (super) blocks each subgroup processes
54
+ // each thread in a subgroup processes a block (32 weights)
55
+ #define BLOCK_STRIDE (N_SIMDWIDTH/8)
56
+
57
+ #ifdef INTEL_GPU
58
+ REQD_SUBGROUP_SIZE_16
59
+ #elif defined (ADRENO_GPU)
60
+ REQD_SUBGROUP_SIZE_64
61
+ #endif
62
+ kernel void kernel_mul_mv_q4_K_f32_flat(
63
+ global uchar * src0_q,
64
+ global uchar * src0_s,
65
+ global half * src0_d,
66
+ global half * src0_dm,
67
+ global char * src1,
68
+ int offset1,
69
+ global char * dst,
70
+ int offsetd,
71
+ int ne00,
72
+ int ne01,
73
+ ulong nb01,
74
+ ulong nb02,
75
+ ulong nb03,
76
+ int ne12,
77
+ ulong nb11,
78
+ ulong nb12,
79
+ ulong nb13,
80
+ int ne0,
81
+ int ne1,
82
+ int r2,
83
+ int r3
84
+ ) {
85
+ src1 = src1 + offset1;
86
+ dst = dst + offsetd;
87
+
88
+ ushort kmask1 = 0x3f3f;
89
+ ushort kmask2 = 0x0f0f;
90
+ ushort kmask3 = 0xc0c0;
91
+
92
+ int ix = get_sub_group_local_id()/8;
93
+ int it = get_sub_group_local_id()%8;
94
+ int iq = it/4;
95
+ int ir = it%4;
96
+
97
+ int nb = ne00/QK_K;
98
+
99
+ int r0 = get_group_id(0);
100
+ int r1 = get_group_id(1);
101
+ int im = get_group_id(2);
102
+ int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;
103
+
104
+ int i12 = im%ne12;
105
+ int i13 = im/ne12;
106
+
107
+ int offset_src0 = (first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03)/BLOCK_Q4K_SIZE;
108
+ uint blk = nb01 / BLOCK_Q4K_SIZE;
109
+ global uchar * blk_q = (global uchar *)src0_q + offset_src0*(QK_K/2);
110
+ global uchar * blk_s = (global uchar *)src0_s + offset_src0*K_SCALE_SIZE;
111
+ global half * blk_d = (global half *)src0_d + offset_src0;
112
+ global half * blk_dm = (global half *)src0_dm + offset_src0;
113
+
114
+ int offset_src1 = r1*nb11 + (i12)*nb12 + (i13)*nb13;
115
+ global float * y = (global float *)(src1 + offset_src1);
116
+
117
+ float yl[16];
118
+ float yh[16];
119
+ float sumf[N_DST] = {0.f};
120
+ float all_sum;
121
+
122
+ global float * y4 = y + ix * QK_K + 64 * iq + 8 * ir;
123
+
124
+ ushort sc16[4];
125
+ uchar * sc8 = (uchar *)sc16;
126
+
127
+ for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) {
128
+ float4 sumy = {0.f, 0.f, 0.f, 0.f};
129
+ for (int i = 0; i < 8; ++i) {
130
+ yl[i+0] = y4[i+0];
131
+ sumy.s0 += yl[i+0];
132
+
133
+ yl[i+8] = y4[i+32];
134
+ sumy.s1 += yl[i+8];
135
+
136
+ yh[i+0] = y4[i+128];
137
+ sumy.s2 += yh[i+0];
138
+
139
+ yh[i+8] = y4[i+160];
140
+ sumy.s3 += yh[i+8];
141
+ }
142
+
143
+ global ushort * q1 = (global ushort *)(blk_q + ib * (QK_K/2)) + (16 * iq + 4 * ir);
144
+ global ushort * sc = (global ushort *)(blk_s + ib * K_SCALE_SIZE) + iq;
145
+ global half * d = blk_d + ib;
146
+ global half * dm = blk_dm + ib;
147
+
148
+ for (int row = 0; row < N_DST; row++) {
149
+ sc16[0] = sc[0] & kmask1;
150
+ sc16[1] = sc[2] & kmask1;
151
+ sc16[2] = ((sc[4] >> 0) & kmask2) | ((sc[0] & kmask3) >> 2);
152
+ sc16[3] = ((sc[4] >> 4) & kmask2) | ((sc[2] & kmask3) >> 2);
153
+
154
+ global ushort * q2 = q1 + 32;
155
+
156
+ float4 acc1 = {0.f, 0.f, 0.f, 0.f};
157
+ float4 acc2 = {0.f, 0.f, 0.f, 0.f};
158
+ for (int i = 0; i < 8; i += 2) {
159
+ acc1.s0 += yl[i+0] * (q1[i/2] & 0x000F);
160
+ acc1.s1 += yl[i+1] * (q1[i/2] & 0x0F00);
161
+ acc1.s2 += yl[i+8] * (q1[i/2] & 0x00F0);
162
+ acc1.s3 += yl[i+9] * (q1[i/2] & 0xF000);
163
+ acc2.s0 += yh[i+0] * (q2[i/2] & 0x000F);
164
+ acc2.s1 += yh[i+1] * (q2[i/2] & 0x0F00);
165
+ acc2.s2 += yh[i+8] * (q2[i/2] & 0x00F0);
166
+ acc2.s3 += yh[i+9] * (q2[i/2] & 0xF000);
167
+ }
168
+
169
+ float dall = *d;
170
+ float dmin = *dm;
171
+ sumf[row] += dall * ((acc1.s0 + 1.f/256.f * acc1.s1) * sc8[0] +
172
+ (acc1.s2 + 1.f/256.f * acc1.s3) * sc8[1] * 1.f/16.f +
173
+ (acc2.s0 + 1.f/256.f * acc2.s1) * sc8[4] +
174
+ (acc2.s2 + 1.f/256.f * acc2.s3) * sc8[5] * 1.f/16.f) -
175
+ dmin * (sumy.s0 * sc8[2] + sumy.s1 * sc8[3] + sumy.s2 * sc8[6] + sumy.s3 * sc8[7]);
176
+
177
+ q1 += blk*64;
178
+ sc += blk*6;
179
+ d += blk;
180
+ dm += blk;
181
+ }
182
+
183
+ y4 += BLOCK_STRIDE * QK_K;
184
+ }
185
+
186
+ global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0;
187
+
188
+ for (int row = 0; row < N_DST; ++row) {
189
+ all_sum = sub_group_reduce_add(sumf[row]);
190
+ if (first_row + row < ne01) {
191
+ if (get_sub_group_local_id() == 0) {
192
+ dst_f32[first_row + row] = all_sum;
193
+ }
194
+ }
195
+ }
196
+ }
llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32.cl ADDED
@@ -0,0 +1,241 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #pragma OPENCL EXTENSION cl_khr_fp16 : enable
2
+
3
+ #ifdef cl_intel_subgroups
4
+ #pragma OPENCL EXTENSION cl_intel_subgroups : enable
5
+ #else
6
+ #pragma OPENCL EXTENSION cl_khr_subgroups : enable
7
+ #endif
8
+
9
+ #ifdef cl_intel_required_subgroup_size
10
+ #pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
11
+ #define INTEL_GPU 1
12
+ #define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
13
+ #define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
14
+ #elif defined(cl_qcom_reqd_sub_group_size)
15
+ #pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
16
+ #define ADRENO_GPU 1
17
+ #define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
18
+ #define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
19
+ #endif
20
+
21
+ #define QK5_0 32
22
+
23
+ struct block_q5_0 {
24
+ half d;
25
+ uchar qh[4];
26
+ uchar qs[QK5_0 / 2];
27
+ };
28
+
29
+ inline float block_q5_0_dot_y(
30
+ global const struct block_q5_0 * qb_curr,
31
+ float sumy,
32
+ float16 yl,
33
+ int il,
34
+ global const float * yb
35
+ ) {
36
+ float d = qb_curr->d;
37
+
38
+ float4 acc = (float4)(0.0f, 0.0f, 0.0f, 0.0f);
39
+
40
+ global const ushort * qs = ((global const ushort *)((global const uchar *) qb_curr + 6 + il));
41
+
42
+ acc.s0 += yl.s0 * (qs[0] & 0x000F);
43
+ acc.s0 += yl.s1 * (qs[0] & 0x0F00);
44
+ acc.s0 += yl.s8 * (qs[0] & 0x00F0);
45
+ acc.s3 += yl.s9 * (qs[0] & 0xF000);
46
+
47
+ acc.s0 += yl.s2 * (qs[1] & 0x000F);
48
+ acc.s1 += yl.s3 * (qs[1] & 0x0F00);
49
+ acc.s2 += yl.sa * (qs[1] & 0x00F0);
50
+ acc.s3 += yl.sb * (qs[1] & 0xF000);
51
+
52
+ acc.s0 += yl.s4 * (qs[2] & 0x000F);
53
+ acc.s1 += yl.s5 * (qs[2] & 0x0F00);
54
+ acc.s2 += yl.sc * (qs[2] & 0x00F0);
55
+ acc.s3 += yl.sd * (qs[2] & 0xF000);
56
+
57
+ acc.s0 += yl.s6 * (qs[3] & 0x000F);
58
+ acc.s1 += yl.s7 * (qs[3] & 0x0F00);
59
+ acc.s2 += yl.se * (qs[3] & 0x00F0);
60
+ acc.s3 += yl.sf * (qs[3] & 0xF000);
61
+
62
+ uint qh_val = *((global const uint *)((global const uchar *) qb_curr + 2));
63
+ uchar qh_lo = (uchar)((qh_val >> il) & 0xFF);
64
+ uchar qh_hi = (uchar)((qh_val >> (il + 16)) & 0xFF);
65
+
66
+ float qh_sum = 0.0f;
67
+ qh_sum += yb[0] * (float)((qh_lo >> 0) & 1);
68
+ qh_sum += yb[1] * (float)((qh_lo >> 1) & 1);
69
+ qh_sum += yb[2] * (float)((qh_lo >> 2) & 1);
70
+ qh_sum += yb[3] * (float)((qh_lo >> 3) & 1);
71
+ qh_sum += yb[4] * (float)((qh_lo >> 4) & 1);
72
+ qh_sum += yb[5] * (float)((qh_lo >> 5) & 1);
73
+ qh_sum += yb[6] * (float)((qh_lo >> 6) & 1);
74
+ qh_sum += yb[7] * (float)((qh_lo >> 7) & 1);
75
+ qh_sum += yb[16] * (float)((qh_hi >> 0) & 1);
76
+ qh_sum += yb[17] * (float)((qh_hi >> 1) & 1);
77
+ qh_sum += yb[18] * (float)((qh_hi >> 2) & 1);
78
+ qh_sum += yb[19] * (float)((qh_hi >> 3) & 1);
79
+ qh_sum += yb[20] * (float)((qh_hi >> 4) & 1);
80
+ qh_sum += yb[21] * (float)((qh_hi >> 5) & 1);
81
+ qh_sum += yb[22] * (float)((qh_hi >> 6) & 1);
82
+ qh_sum += yb[23] * (float)((qh_hi >> 7) & 1);
83
+
84
+ return d * (acc.s0 + acc.s1 + acc.s2 + acc.s3 + 16.0f * qh_sum - 16.0f * sumy);
85
+ }
86
+
87
+ #undef N_DST
88
+ #undef N_SIMDGROUP
89
+ #undef N_SIMDWIDTH
90
+
91
+ #ifdef INTEL_GPU
92
+ #define N_DST 4 // each subgroup works on 4 rows
93
+ #define N_SIMDGROUP 1 // number of subgroups in a thread group
94
+ #define N_SIMDWIDTH 16 // assuming subgroup size is 16
95
+ #elif defined (ADRENO_GPU)
96
+ #define N_DST 4
97
+ #define N_SIMDGROUP 1
98
+ #define N_SIMDWIDTH 64
99
+ #endif
100
+
101
+ inline void mul_vec_q_n_f32(
102
+ global void * src0,
103
+ global float * src1,
104
+ global float * dst,
105
+ int ne00,
106
+ int ne01,
107
+ int ne02,
108
+ int ne10,
109
+ int ne12,
110
+ int ne0,
111
+ int ne1,
112
+ int r2,
113
+ int r3
114
+ ) {
115
+ const ulong nb = ne00/QK5_0;
116
+
117
+ int r0 = get_group_id(0);
118
+ int r1 = get_group_id(1);
119
+ int im = get_group_id(2);
120
+
121
+ int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;
122
+
123
+ int i12 = im%ne12;
124
+ int i13 = im/ne12;
125
+
126
+ ulong offset0 = first_row * nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);
127
+
128
+ global struct block_q5_0 * x = (global struct block_q5_0 *) src0 + offset0;
129
+ global float * y = (global float *) src1 + r1*ne10 + im*ne00*ne1;
130
+
131
+ float16 yl;
132
+ float4 sumf = (float4)(0.f, 0.f, 0.f, 0.f);
133
+
134
+ int ix = get_sub_group_local_id()/2;
135
+ int il = 8*(get_sub_group_local_id()%2);
136
+
137
+ global float * yb = y + ix * QK5_0 + il;
138
+
139
+ for (int ib = ix; ib < nb; ib += N_SIMDWIDTH/2) {
140
+ float sumy = 0;
141
+
142
+ sumy += yb[0];
143
+ sumy += yb[1];
144
+ sumy += yb[2];
145
+ sumy += yb[3];
146
+ sumy += yb[4];
147
+ sumy += yb[5];
148
+ sumy += yb[6];
149
+ sumy += yb[7];
150
+
151
+ sumy += yb[16];
152
+ sumy += yb[17];
153
+ sumy += yb[18];
154
+ sumy += yb[19];
155
+ sumy += yb[20];
156
+ sumy += yb[21];
157
+ sumy += yb[22];
158
+ sumy += yb[23];
159
+
160
+
161
+ yl.s0 = yb[0];
162
+ yl.s1 = yb[1]/256.f;
163
+
164
+ yl.s2 = yb[2];
165
+ yl.s3 = yb[3]/256.f;
166
+
167
+ yl.s4 = yb[4];
168
+ yl.s5 = yb[5]/256.f;
169
+
170
+ yl.s6 = yb[6];
171
+ yl.s7 = yb[7]/256.f;
172
+
173
+ yl.s8 = yb[16]/16.f;
174
+ yl.s9 = yb[17]/4096.f;
175
+
176
+ yl.sa = yb[18]/16.f;
177
+ yl.sb = yb[19]/4096.f;
178
+
179
+ yl.sc = yb[20]/16.f;
180
+ yl.sd = yb[21]/4096.f;
181
+
182
+ yl.se = yb[22]/16.f;
183
+ yl.sf = yb[23]/4096.f;
184
+
185
+ sumf.s0 += block_q5_0_dot_y(x+ib+0*nb, sumy, yl, il, yb);
186
+ sumf.s1 += block_q5_0_dot_y(x+ib+1*nb, sumy, yl, il, yb);
187
+ sumf.s2 += block_q5_0_dot_y(x+ib+2*nb, sumy, yl, il, yb);
188
+ sumf.s3 += block_q5_0_dot_y(x+ib+3*nb, sumy, yl, il, yb);
189
+
190
+ yb += QK5_0 * (N_SIMDWIDTH/2);
191
+ }
192
+
193
+ float4 tot = (float4)(
194
+ sub_group_reduce_add(sumf.s0), sub_group_reduce_add(sumf.s1),
195
+ sub_group_reduce_add(sumf.s2), sub_group_reduce_add(sumf.s3)
196
+ );
197
+
198
+ if (get_sub_group_local_id() == 0) {
199
+ if (first_row + 0 < ne01) {
200
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 0] = tot.s0;
201
+ }
202
+ if (first_row + 1 < ne01) {
203
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 1] = tot.s1;
204
+ }
205
+ if (first_row + 2 < ne01) {
206
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 2] = tot.s2;
207
+ }
208
+ if (first_row + 3 < ne01) {
209
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 3] = tot.s3;
210
+ }
211
+ }
212
+ }
213
+
214
+ #ifdef INTEL_GPU
215
+ REQD_SUBGROUP_SIZE_16
216
+ #elif defined (ADRENO_GPU)
217
+ REQD_SUBGROUP_SIZE_64
218
+ #endif
219
+ kernel void kernel_mul_mv_q5_0_f32(
220
+ global void * src0,
221
+ ulong offset0,
222
+ global float * src1,
223
+ ulong offset1,
224
+ global float * dst,
225
+ ulong offsetd,
226
+ int ne00,
227
+ int ne01,
228
+ int ne02,
229
+ int ne10,
230
+ int ne12,
231
+ int ne0,
232
+ int ne1,
233
+ int r2,
234
+ int r3
235
+ ) {
236
+ src0 = (global void*)((global char*)src0 + offset0);
237
+ src1 = (global float*)((global char*)src1 + offset1);
238
+ dst = (global float*)((global char*)dst + offsetd);
239
+
240
+ mul_vec_q_n_f32(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3);
241
+ }
llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32_flat.cl ADDED
@@ -0,0 +1,243 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ #pragma OPENCL EXTENSION cl_khr_fp16 : enable
3
+
4
+ #ifdef cl_intel_subgroups
5
+ #pragma OPENCL EXTENSION cl_intel_subgroups : enable
6
+ #else
7
+ #pragma OPENCL EXTENSION cl_khr_subgroups : enable
8
+ #endif
9
+
10
+ #ifdef cl_intel_required_subgroup_size
11
+ #pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
12
+ #define INTEL_GPU 1
13
+ #define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
14
+ #define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
15
+ #elif defined(cl_qcom_reqd_sub_group_size)
16
+ #pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
17
+ #define ADRENO_GPU 1
18
+ #define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
19
+ #define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
20
+ #endif
21
+
22
+ #define QK5_0 32
23
+
24
+ inline float block_q5_0_dot_y_flat(
25
+ global const uchar * x,
26
+ global const uint * qh_ptr,
27
+ global const half * dh,
28
+ float sumy,
29
+ float16 yl,
30
+ int il,
31
+ global const float * yb
32
+ ) {
33
+ float d = *dh;
34
+ global const ushort * qs = ((global const ushort *)(x + il));
35
+
36
+ float4 acc = (float4)(0.0f, 0.0f, 0.0f, 0.0f);
37
+
38
+ acc.s0 += yl.s0 * (qs[0] & 0x000F);
39
+ acc.s0 += yl.s1 * (qs[0] & 0x0F00);
40
+ acc.s0 += yl.s8 * (qs[0] & 0x00F0);
41
+ acc.s3 += yl.s9 * (qs[0] & 0xF000);
42
+
43
+ acc.s0 += yl.s2 * (qs[1] & 0x000F);
44
+ acc.s1 += yl.s3 * (qs[1] & 0x0F00);
45
+ acc.s2 += yl.sa * (qs[1] & 0x00F0);
46
+ acc.s3 += yl.sb * (qs[1] & 0xF000);
47
+
48
+ acc.s0 += yl.s4 * (qs[2] & 0x000F);
49
+ acc.s1 += yl.s5 * (qs[2] & 0x0F00);
50
+ acc.s2 += yl.sc * (qs[2] & 0x00F0);
51
+ acc.s3 += yl.sd * (qs[2] & 0xF000);
52
+
53
+ acc.s0 += yl.s6 * (qs[3] & 0x000F);
54
+ acc.s1 += yl.s7 * (qs[3] & 0x0F00);
55
+ acc.s2 += yl.se * (qs[3] & 0x00F0);
56
+ acc.s3 += yl.sf * (qs[3] & 0xF000);
57
+
58
+ uint qh_val = *qh_ptr;
59
+ uchar qh_lo = (uchar)((qh_val >> il) & 0xFF);
60
+ uchar qh_hi = (uchar)((qh_val >> (il + 16)) & 0xFF);
61
+
62
+ float qh_sum = 0.0f;
63
+ qh_sum += yb[0] * (float)((qh_lo >> 0) & 1);
64
+ qh_sum += yb[1] * (float)((qh_lo >> 1) & 1);
65
+ qh_sum += yb[2] * (float)((qh_lo >> 2) & 1);
66
+ qh_sum += yb[3] * (float)((qh_lo >> 3) & 1);
67
+ qh_sum += yb[4] * (float)((qh_lo >> 4) & 1);
68
+ qh_sum += yb[5] * (float)((qh_lo >> 5) & 1);
69
+ qh_sum += yb[6] * (float)((qh_lo >> 6) & 1);
70
+ qh_sum += yb[7] * (float)((qh_lo >> 7) & 1);
71
+ qh_sum += yb[16] * (float)((qh_hi >> 0) & 1);
72
+ qh_sum += yb[17] * (float)((qh_hi >> 1) & 1);
73
+ qh_sum += yb[18] * (float)((qh_hi >> 2) & 1);
74
+ qh_sum += yb[19] * (float)((qh_hi >> 3) & 1);
75
+ qh_sum += yb[20] * (float)((qh_hi >> 4) & 1);
76
+ qh_sum += yb[21] * (float)((qh_hi >> 5) & 1);
77
+ qh_sum += yb[22] * (float)((qh_hi >> 6) & 1);
78
+ qh_sum += yb[23] * (float)((qh_hi >> 7) & 1);
79
+
80
+ return d * (acc.s0 + acc.s1 + acc.s2 + acc.s3 + 16.0f * qh_sum - 16.0f * sumy);
81
+ }
82
+
83
+ #undef N_DST
84
+ #undef N_SIMDGROUP
85
+ #undef N_SIMDWIDTH
86
+
87
+ #ifdef INTEL_GPU
88
+ #define N_DST 4 // each subgroup works on 4 rows
89
+ #define N_SIMDGROUP 1 // number of subgroups in a thread group
90
+ #define N_SIMDWIDTH 16 // assuming subgroup size is 16
91
+ #elif defined (ADRENO_GPU)
92
+ #define N_DST 4
93
+ #define N_SIMDGROUP 1
94
+ #define N_SIMDWIDTH 64
95
+ #endif
96
+
97
+ inline void mul_vec_q_n_f32_flat(
98
+ global void * src0_qs,
99
+ global void * src0_qh,
100
+ global void * src0_d,
101
+ global float * src1,
102
+ global float * dst,
103
+ int ne00,
104
+ int ne01,
105
+ int ne02,
106
+ int ne10,
107
+ int ne12,
108
+ int ne0,
109
+ int ne1,
110
+ int r2,
111
+ int r3
112
+ ) {
113
+ const ulong nb = ne00/QK5_0;
114
+
115
+ int r0 = get_group_id(0);
116
+ int r1 = get_group_id(1);
117
+ int im = get_group_id(2);
118
+
119
+ int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;
120
+
121
+ int i12 = im%ne12;
122
+ int i13 = im/ne12;
123
+
124
+ ulong offset0 = first_row * nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);
125
+
126
+ ulong offset0_qs = offset0 * (QK5_0/2);
127
+
128
+ global uchar * x = (global uchar *) src0_qs + offset0_qs;
129
+ global uint * qh = (global uint *) src0_qh + offset0;
130
+ global half * d = (global half *) src0_d + offset0;
131
+ global float * y = (global float *) src1 + r1*ne10 + im*ne00*ne1;
132
+
133
+ float16 yl;
134
+ float4 sumf = (float4)(0.f, 0.f, 0.f, 0.f);
135
+
136
+ int ix = get_sub_group_local_id()/2;
137
+ int il = 8*(get_sub_group_local_id()%2);
138
+
139
+ global float * yb = y + ix * QK5_0 + il;
140
+
141
+ for (int ib = ix; ib < nb; ib += N_SIMDWIDTH/2) {
142
+ float sumy = 0;
143
+
144
+ sumy += yb[0];
145
+ sumy += yb[1];
146
+ sumy += yb[2];
147
+ sumy += yb[3];
148
+ sumy += yb[4];
149
+ sumy += yb[5];
150
+ sumy += yb[6];
151
+ sumy += yb[7];
152
+
153
+ sumy += yb[16];
154
+ sumy += yb[17];
155
+ sumy += yb[18];
156
+ sumy += yb[19];
157
+ sumy += yb[20];
158
+ sumy += yb[21];
159
+ sumy += yb[22];
160
+ sumy += yb[23];
161
+
162
+
163
+ yl.s0 = yb[0];
164
+ yl.s1 = yb[1]/256.f;
165
+
166
+ yl.s2 = yb[2];
167
+ yl.s3 = yb[3]/256.f;
168
+
169
+ yl.s4 = yb[4];
170
+ yl.s5 = yb[5]/256.f;
171
+
172
+ yl.s6 = yb[6];
173
+ yl.s7 = yb[7]/256.f;
174
+
175
+ yl.s8 = yb[16]/16.f;
176
+ yl.s9 = yb[17]/4096.f;
177
+
178
+ yl.sa = yb[18]/16.f;
179
+ yl.sb = yb[19]/4096.f;
180
+
181
+ yl.sc = yb[20]/16.f;
182
+ yl.sd = yb[21]/4096.f;
183
+
184
+ yl.se = yb[22]/16.f;
185
+ yl.sf = yb[23]/4096.f;
186
+
187
+ sumf.s0 += block_q5_0_dot_y_flat(x + ib*(QK5_0/2) + 0*nb*(QK5_0/2), qh + ib + 0*nb, d + ib + 0*nb, sumy, yl, il, yb);
188
+ sumf.s1 += block_q5_0_dot_y_flat(x + ib*(QK5_0/2) + 1*nb*(QK5_0/2), qh + ib + 1*nb, d + ib + 1*nb, sumy, yl, il, yb);
189
+ sumf.s2 += block_q5_0_dot_y_flat(x + ib*(QK5_0/2) + 2*nb*(QK5_0/2), qh + ib + 2*nb, d + ib + 2*nb, sumy, yl, il, yb);
190
+ sumf.s3 += block_q5_0_dot_y_flat(x + ib*(QK5_0/2) + 3*nb*(QK5_0/2), qh + ib + 3*nb, d + ib + 3*nb, sumy, yl, il, yb);
191
+
192
+ yb += QK5_0 * (N_SIMDWIDTH/2);
193
+ }
194
+
195
+ float4 tot = (float4)(
196
+ sub_group_reduce_add(sumf.s0), sub_group_reduce_add(sumf.s1),
197
+ sub_group_reduce_add(sumf.s2), sub_group_reduce_add(sumf.s3)
198
+ );
199
+
200
+ if (get_sub_group_local_id() == 0) {
201
+ if (first_row + 0 < ne01) {
202
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 0] = tot.s0;
203
+ }
204
+ if (first_row + 1 < ne01) {
205
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 1] = tot.s1;
206
+ }
207
+ if (first_row + 2 < ne01) {
208
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 2] = tot.s2;
209
+ }
210
+ if (first_row + 3 < ne01) {
211
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 3] = tot.s3;
212
+ }
213
+ }
214
+ }
215
+
216
+ #ifdef INTEL_GPU
217
+ REQD_SUBGROUP_SIZE_16
218
+ #elif defined (ADRENO_GPU)
219
+ REQD_SUBGROUP_SIZE_64
220
+ #endif
221
+ kernel void kernel_mul_mv_q5_0_f32_flat(
222
+ global void * src0_qs,
223
+ global void * src0_qh,
224
+ global void * src0_d,
225
+ global float * src1,
226
+ ulong offset1,
227
+ global float * dst,
228
+ ulong offsetd,
229
+ int ne00,
230
+ int ne01,
231
+ int ne02,
232
+ int ne10,
233
+ int ne12,
234
+ int ne0,
235
+ int ne1,
236
+ int r2,
237
+ int r3
238
+ ) {
239
+ src1 = (global float*)((global char*)src1 + offset1);
240
+ dst = (global float*)((global char*)dst + offsetd);
241
+
242
+ mul_vec_q_n_f32_flat(src0_qs, src0_qh, src0_d, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3);
243
+ }
llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32.cl ADDED
@@ -0,0 +1,243 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #pragma OPENCL EXTENSION cl_khr_fp16 : enable
2
+
3
+ #ifdef cl_intel_subgroups
4
+ #pragma OPENCL EXTENSION cl_intel_subgroups : enable
5
+ #else
6
+ #pragma OPENCL EXTENSION cl_khr_subgroups : enable
7
+ #endif
8
+
9
+ #ifdef cl_intel_required_subgroup_size
10
+ #pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
11
+ #define INTEL_GPU 1
12
+ #define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
13
+ #define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
14
+ #elif defined(cl_qcom_reqd_sub_group_size)
15
+ #pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
16
+ #define ADRENO_GPU 1
17
+ #define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
18
+ #define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
19
+ #endif
20
+
21
+ #define QK5_1 32
22
+
23
+ struct block_q5_1 {
24
+ half d;
25
+ half m;
26
+ uchar qh[4];
27
+ uchar qs[QK5_1 / 2];
28
+ };
29
+
30
+ inline float block_q5_1_dot_y(
31
+ global const struct block_q5_1 * qb_curr,
32
+ float sumy,
33
+ float16 yl,
34
+ int il,
35
+ global const float * yb
36
+ ) {
37
+ float d = qb_curr->d;
38
+ float m = qb_curr->m;
39
+
40
+ float4 acc = (float4)(0.0f, 0.0f, 0.0f, 0.0f);
41
+
42
+ global const ushort * qs = ((global const ushort *)((global const uchar *) qb_curr + 8 + il));
43
+
44
+ acc.s0 += yl.s0 * (qs[0] & 0x000F);
45
+ acc.s0 += yl.s1 * (qs[0] & 0x0F00);
46
+ acc.s0 += yl.s8 * (qs[0] & 0x00F0);
47
+ acc.s3 += yl.s9 * (qs[0] & 0xF000);
48
+
49
+ acc.s0 += yl.s2 * (qs[1] & 0x000F);
50
+ acc.s1 += yl.s3 * (qs[1] & 0x0F00);
51
+ acc.s2 += yl.sa * (qs[1] & 0x00F0);
52
+ acc.s3 += yl.sb * (qs[1] & 0xF000);
53
+
54
+ acc.s0 += yl.s4 * (qs[2] & 0x000F);
55
+ acc.s1 += yl.s5 * (qs[2] & 0x0F00);
56
+ acc.s2 += yl.sc * (qs[2] & 0x00F0);
57
+ acc.s3 += yl.sd * (qs[2] & 0xF000);
58
+
59
+ acc.s0 += yl.s6 * (qs[3] & 0x000F);
60
+ acc.s1 += yl.s7 * (qs[3] & 0x0F00);
61
+ acc.s2 += yl.se * (qs[3] & 0x00F0);
62
+ acc.s3 += yl.sf * (qs[3] & 0xF000);
63
+
64
+ uint qh_val = *((global const uint *)((global const uchar *) qb_curr + 4));
65
+ uchar qh_lo = (uchar)((qh_val >> il) & 0xFF);
66
+ uchar qh_hi = (uchar)((qh_val >> (il + 16)) & 0xFF);
67
+
68
+ float qh_sum = 0.0f;
69
+ qh_sum += yb[0] * (float)((qh_lo >> 0) & 1);
70
+ qh_sum += yb[1] * (float)((qh_lo >> 1) & 1);
71
+ qh_sum += yb[2] * (float)((qh_lo >> 2) & 1);
72
+ qh_sum += yb[3] * (float)((qh_lo >> 3) & 1);
73
+ qh_sum += yb[4] * (float)((qh_lo >> 4) & 1);
74
+ qh_sum += yb[5] * (float)((qh_lo >> 5) & 1);
75
+ qh_sum += yb[6] * (float)((qh_lo >> 6) & 1);
76
+ qh_sum += yb[7] * (float)((qh_lo >> 7) & 1);
77
+ qh_sum += yb[16] * (float)((qh_hi >> 0) & 1);
78
+ qh_sum += yb[17] * (float)((qh_hi >> 1) & 1);
79
+ qh_sum += yb[18] * (float)((qh_hi >> 2) & 1);
80
+ qh_sum += yb[19] * (float)((qh_hi >> 3) & 1);
81
+ qh_sum += yb[20] * (float)((qh_hi >> 4) & 1);
82
+ qh_sum += yb[21] * (float)((qh_hi >> 5) & 1);
83
+ qh_sum += yb[22] * (float)((qh_hi >> 6) & 1);
84
+ qh_sum += yb[23] * (float)((qh_hi >> 7) & 1);
85
+
86
+ return d * (acc.s0 + acc.s1 + acc.s2 + acc.s3 + 16.0f * qh_sum) + sumy * m;
87
+ }
88
+
89
+ #undef N_DST
90
+ #undef N_SIMDGROUP
91
+ #undef N_SIMDWIDTH
92
+
93
+ #ifdef INTEL_GPU
94
+ #define N_DST 4 // each subgroup works on 4 rows
95
+ #define N_SIMDGROUP 1 // number of subgroups in a thread group
96
+ #define N_SIMDWIDTH 16 // assuming subgroup size is 16
97
+ #elif defined (ADRENO_GPU)
98
+ #define N_DST 4
99
+ #define N_SIMDGROUP 1
100
+ #define N_SIMDWIDTH 64
101
+ #endif
102
+
103
+ inline void mul_vec_q_n_f32(
104
+ global void * src0,
105
+ global float * src1,
106
+ global float * dst,
107
+ int ne00,
108
+ int ne01,
109
+ int ne02,
110
+ int ne10,
111
+ int ne12,
112
+ int ne0,
113
+ int ne1,
114
+ int r2,
115
+ int r3
116
+ ) {
117
+ const ulong nb = ne00/QK5_1;
118
+
119
+ int r0 = get_group_id(0);
120
+ int r1 = get_group_id(1);
121
+ int im = get_group_id(2);
122
+
123
+ int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;
124
+
125
+ int i12 = im%ne12;
126
+ int i13 = im/ne12;
127
+
128
+ ulong offset0 = first_row * nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);
129
+
130
+ global struct block_q5_1 * x = (global struct block_q5_1 *) src0 + offset0;
131
+ global float * y = (global float *) src1 + r1*ne10 + im*ne00*ne1;
132
+
133
+ float16 yl;
134
+ float4 sumf = (float4)(0.f, 0.f, 0.f, 0.f);
135
+
136
+ int ix = get_sub_group_local_id()/2;
137
+ int il = 8*(get_sub_group_local_id()%2);
138
+
139
+ global float * yb = y + ix * QK5_1 + il;
140
+
141
+ for (int ib = ix; ib < nb; ib += N_SIMDWIDTH/2) {
142
+ float sumy = 0;
143
+
144
+ sumy += yb[0];
145
+ sumy += yb[1];
146
+ sumy += yb[2];
147
+ sumy += yb[3];
148
+ sumy += yb[4];
149
+ sumy += yb[5];
150
+ sumy += yb[6];
151
+ sumy += yb[7];
152
+
153
+ sumy += yb[16];
154
+ sumy += yb[17];
155
+ sumy += yb[18];
156
+ sumy += yb[19];
157
+ sumy += yb[20];
158
+ sumy += yb[21];
159
+ sumy += yb[22];
160
+ sumy += yb[23];
161
+
162
+
163
+ yl.s0 = yb[0];
164
+ yl.s1 = yb[1]/256.f;
165
+
166
+ yl.s2 = yb[2];
167
+ yl.s3 = yb[3]/256.f;
168
+
169
+ yl.s4 = yb[4];
170
+ yl.s5 = yb[5]/256.f;
171
+
172
+ yl.s6 = yb[6];
173
+ yl.s7 = yb[7]/256.f;
174
+
175
+ yl.s8 = yb[16]/16.f;
176
+ yl.s9 = yb[17]/4096.f;
177
+
178
+ yl.sa = yb[18]/16.f;
179
+ yl.sb = yb[19]/4096.f;
180
+
181
+ yl.sc = yb[20]/16.f;
182
+ yl.sd = yb[21]/4096.f;
183
+
184
+ yl.se = yb[22]/16.f;
185
+ yl.sf = yb[23]/4096.f;
186
+
187
+ sumf.s0 += block_q5_1_dot_y(x+ib+0*nb, sumy, yl, il, yb);
188
+ sumf.s1 += block_q5_1_dot_y(x+ib+1*nb, sumy, yl, il, yb);
189
+ sumf.s2 += block_q5_1_dot_y(x+ib+2*nb, sumy, yl, il, yb);
190
+ sumf.s3 += block_q5_1_dot_y(x+ib+3*nb, sumy, yl, il, yb);
191
+
192
+ yb += QK5_1 * (N_SIMDWIDTH/2);
193
+ }
194
+
195
+ float4 tot = (float4)(
196
+ sub_group_reduce_add(sumf.s0), sub_group_reduce_add(sumf.s1),
197
+ sub_group_reduce_add(sumf.s2), sub_group_reduce_add(sumf.s3)
198
+ );
199
+
200
+ if (get_sub_group_local_id() == 0) {
201
+ if (first_row + 0 < ne01) {
202
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 0] = tot.s0;
203
+ }
204
+ if (first_row + 1 < ne01) {
205
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 1] = tot.s1;
206
+ }
207
+ if (first_row + 2 < ne01) {
208
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 2] = tot.s2;
209
+ }
210
+ if (first_row + 3 < ne01) {
211
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 3] = tot.s3;
212
+ }
213
+ }
214
+ }
215
+
216
+ #ifdef INTEL_GPU
217
+ REQD_SUBGROUP_SIZE_16
218
+ #elif defined (ADRENO_GPU)
219
+ REQD_SUBGROUP_SIZE_64
220
+ #endif
221
+ kernel void kernel_mul_mv_q5_1_f32(
222
+ global void * src0,
223
+ ulong offset0,
224
+ global float * src1,
225
+ ulong offset1,
226
+ global float * dst,
227
+ ulong offsetd,
228
+ int ne00,
229
+ int ne01,
230
+ int ne02,
231
+ int ne10,
232
+ int ne12,
233
+ int ne0,
234
+ int ne1,
235
+ int r2,
236
+ int r3
237
+ ) {
238
+ src0 = (global void*)((global char*)src0 + offset0);
239
+ src1 = (global float*)((global char*)src1 + offset1);
240
+ dst = (global float*)((global char*)dst + offsetd);
241
+
242
+ mul_vec_q_n_f32(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3);
243
+ }
llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32_flat.cl ADDED
@@ -0,0 +1,247 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #pragma OPENCL EXTENSION cl_khr_fp16 : enable
2
+
3
+ #ifdef cl_intel_subgroups
4
+ #pragma OPENCL EXTENSION cl_intel_subgroups : enable
5
+ #else
6
+ #pragma OPENCL EXTENSION cl_khr_subgroups : enable
7
+ #endif
8
+
9
+ #ifdef cl_intel_required_subgroup_size
10
+ #pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
11
+ #define INTEL_GPU 1
12
+ #define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
13
+ #define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
14
+ #elif defined(cl_qcom_reqd_sub_group_size)
15
+ #pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
16
+ #define ADRENO_GPU 1
17
+ #define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
18
+ #define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
19
+ #endif
20
+
21
+ #define QK5_1 32
22
+
23
+ inline float block_q5_1_dot_y_flat(
24
+ global const uchar * x,
25
+ global const uint * qh_ptr,
26
+ global const half * dh,
27
+ global const half * mh,
28
+ float sumy,
29
+ float16 yl,
30
+ int il,
31
+ global const float * yb
32
+ ) {
33
+ float d = *dh;
34
+ float m = *mh;
35
+ global const ushort * qs = ((global const ushort *)(x + il));
36
+
37
+ float4 acc = (float4)(0.0f, 0.0f, 0.0f, 0.0f);
38
+
39
+ acc.s0 += yl.s0 * (qs[0] & 0x000F);
40
+ acc.s0 += yl.s1 * (qs[0] & 0x0F00);
41
+ acc.s0 += yl.s8 * (qs[0] & 0x00F0);
42
+ acc.s3 += yl.s9 * (qs[0] & 0xF000);
43
+
44
+ acc.s0 += yl.s2 * (qs[1] & 0x000F);
45
+ acc.s1 += yl.s3 * (qs[1] & 0x0F00);
46
+ acc.s2 += yl.sa * (qs[1] & 0x00F0);
47
+ acc.s3 += yl.sb * (qs[1] & 0xF000);
48
+
49
+ acc.s0 += yl.s4 * (qs[2] & 0x000F);
50
+ acc.s1 += yl.s5 * (qs[2] & 0x0F00);
51
+ acc.s2 += yl.sc * (qs[2] & 0x00F0);
52
+ acc.s3 += yl.sd * (qs[2] & 0xF000);
53
+
54
+ acc.s0 += yl.s6 * (qs[3] & 0x000F);
55
+ acc.s1 += yl.s7 * (qs[3] & 0x0F00);
56
+ acc.s2 += yl.se * (qs[3] & 0x00F0);
57
+ acc.s3 += yl.sf * (qs[3] & 0xF000);
58
+
59
+ uint qh_val = *qh_ptr;
60
+ uchar qh_lo = (uchar)((qh_val >> il) & 0xFF);
61
+ uchar qh_hi = (uchar)((qh_val >> (il + 16)) & 0xFF);
62
+
63
+ float qh_sum = 0.0f;
64
+ qh_sum += yb[0] * (float)((qh_lo >> 0) & 1);
65
+ qh_sum += yb[1] * (float)((qh_lo >> 1) & 1);
66
+ qh_sum += yb[2] * (float)((qh_lo >> 2) & 1);
67
+ qh_sum += yb[3] * (float)((qh_lo >> 3) & 1);
68
+ qh_sum += yb[4] * (float)((qh_lo >> 4) & 1);
69
+ qh_sum += yb[5] * (float)((qh_lo >> 5) & 1);
70
+ qh_sum += yb[6] * (float)((qh_lo >> 6) & 1);
71
+ qh_sum += yb[7] * (float)((qh_lo >> 7) & 1);
72
+ qh_sum += yb[16] * (float)((qh_hi >> 0) & 1);
73
+ qh_sum += yb[17] * (float)((qh_hi >> 1) & 1);
74
+ qh_sum += yb[18] * (float)((qh_hi >> 2) & 1);
75
+ qh_sum += yb[19] * (float)((qh_hi >> 3) & 1);
76
+ qh_sum += yb[20] * (float)((qh_hi >> 4) & 1);
77
+ qh_sum += yb[21] * (float)((qh_hi >> 5) & 1);
78
+ qh_sum += yb[22] * (float)((qh_hi >> 6) & 1);
79
+ qh_sum += yb[23] * (float)((qh_hi >> 7) & 1);
80
+
81
+ return d * (acc.s0 + acc.s1 + acc.s2 + acc.s3 + 16.0f * qh_sum) + sumy * m;
82
+ }
83
+
84
+ #undef N_DST
85
+ #undef N_SIMDGROUP
86
+ #undef N_SIMDWIDTH
87
+
88
+ #ifdef INTEL_GPU
89
+ #define N_DST 4 // each subgroup works on 4 rows
90
+ #define N_SIMDGROUP 1 // number of subgroups in a thread group
91
+ #define N_SIMDWIDTH 16 // assuming subgroup size is 16
92
+ #elif defined (ADRENO_GPU)
93
+ #define N_DST 4
94
+ #define N_SIMDGROUP 1
95
+ #define N_SIMDWIDTH 64
96
+ #endif
97
+
98
+ inline void mul_vec_q_n_f32_flat(
99
+ global void * src0_qs,
100
+ global void * src0_qh,
101
+ global void * src0_d,
102
+ global void * src0_m,
103
+ global float * src1,
104
+ global float * dst,
105
+ int ne00,
106
+ int ne01,
107
+ int ne02,
108
+ int ne10,
109
+ int ne12,
110
+ int ne0,
111
+ int ne1,
112
+ int r2,
113
+ int r3
114
+ ) {
115
+ const ulong nb = ne00/QK5_1;
116
+
117
+ int r0 = get_group_id(0);
118
+ int r1 = get_group_id(1);
119
+ int im = get_group_id(2);
120
+
121
+ int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;
122
+
123
+ int i12 = im%ne12;
124
+ int i13 = im/ne12;
125
+
126
+ ulong offset0 = first_row * nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);
127
+
128
+ ulong offset0_qs = offset0 * (QK5_1/2);
129
+
130
+ global uchar * x = (global uchar *) src0_qs + offset0_qs;
131
+ global uint * qh = (global uint *) src0_qh + offset0;
132
+ global half * d = (global half *) src0_d + offset0;
133
+ global half * ms = (global half *) src0_m + offset0;
134
+ global float * y = (global float *) src1 + r1*ne10 + im*ne00*ne1;
135
+
136
+ float16 yl;
137
+ float4 sumf = (float4)(0.f, 0.f, 0.f, 0.f);
138
+
139
+ int ix = get_sub_group_local_id()/2;
140
+ int il = 8*(get_sub_group_local_id()%2);
141
+
142
+ global float * yb = y + ix * QK5_1 + il;
143
+
144
+ for (int ib = ix; ib < nb; ib += N_SIMDWIDTH/2) {
145
+ float sumy = 0;
146
+
147
+ sumy += yb[0];
148
+ sumy += yb[1];
149
+ sumy += yb[2];
150
+ sumy += yb[3];
151
+ sumy += yb[4];
152
+ sumy += yb[5];
153
+ sumy += yb[6];
154
+ sumy += yb[7];
155
+
156
+ sumy += yb[16];
157
+ sumy += yb[17];
158
+ sumy += yb[18];
159
+ sumy += yb[19];
160
+ sumy += yb[20];
161
+ sumy += yb[21];
162
+ sumy += yb[22];
163
+ sumy += yb[23];
164
+
165
+
166
+ yl.s0 = yb[0];
167
+ yl.s1 = yb[1]/256.f;
168
+
169
+ yl.s2 = yb[2];
170
+ yl.s3 = yb[3]/256.f;
171
+
172
+ yl.s4 = yb[4];
173
+ yl.s5 = yb[5]/256.f;
174
+
175
+ yl.s6 = yb[6];
176
+ yl.s7 = yb[7]/256.f;
177
+
178
+ yl.s8 = yb[16]/16.f;
179
+ yl.s9 = yb[17]/4096.f;
180
+
181
+ yl.sa = yb[18]/16.f;
182
+ yl.sb = yb[19]/4096.f;
183
+
184
+ yl.sc = yb[20]/16.f;
185
+ yl.sd = yb[21]/4096.f;
186
+
187
+ yl.se = yb[22]/16.f;
188
+ yl.sf = yb[23]/4096.f;
189
+
190
+ sumf.s0 += block_q5_1_dot_y_flat(x + ib*(QK5_1/2) + 0*nb*(QK5_1/2), qh + ib + 0*nb, d + ib + 0*nb, ms + ib + 0*nb, sumy, yl, il, yb);
191
+ sumf.s1 += block_q5_1_dot_y_flat(x + ib*(QK5_1/2) + 1*nb*(QK5_1/2), qh + ib + 1*nb, d + ib + 1*nb, ms + ib + 1*nb, sumy, yl, il, yb);
192
+ sumf.s2 += block_q5_1_dot_y_flat(x + ib*(QK5_1/2) + 2*nb*(QK5_1/2), qh + ib + 2*nb, d + ib + 2*nb, ms + ib + 2*nb, sumy, yl, il, yb);
193
+ sumf.s3 += block_q5_1_dot_y_flat(x + ib*(QK5_1/2) + 3*nb*(QK5_1/2), qh + ib + 3*nb, d + ib + 3*nb, ms + ib + 3*nb, sumy, yl, il, yb);
194
+
195
+ yb += QK5_1 * (N_SIMDWIDTH/2);
196
+ }
197
+
198
+ float4 tot = (float4)(
199
+ sub_group_reduce_add(sumf.s0), sub_group_reduce_add(sumf.s1),
200
+ sub_group_reduce_add(sumf.s2), sub_group_reduce_add(sumf.s3)
201
+ );
202
+
203
+ if (get_sub_group_local_id() == 0) {
204
+ if (first_row + 0 < ne01) {
205
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 0] = tot.s0;
206
+ }
207
+ if (first_row + 1 < ne01) {
208
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 1] = tot.s1;
209
+ }
210
+ if (first_row + 2 < ne01) {
211
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 2] = tot.s2;
212
+ }
213
+ if (first_row + 3 < ne01) {
214
+ dst[r1*ne0 + im*ne0*ne1 + first_row + 3] = tot.s3;
215
+ }
216
+ }
217
+ }
218
+
219
+ #ifdef INTEL_GPU
220
+ REQD_SUBGROUP_SIZE_16
221
+ #elif defined (ADRENO_GPU)
222
+ REQD_SUBGROUP_SIZE_64
223
+ #endif
224
+ kernel void kernel_mul_mv_q5_1_f32_flat(
225
+ global void * src0_qs,
226
+ global void * src0_qh,
227
+ global void * src0_d,
228
+ global void * src0_m,
229
+ global float * src1,
230
+ ulong offset1,
231
+ global float * dst,
232
+ ulong offsetd,
233
+ int ne00,
234
+ int ne01,
235
+ int ne02,
236
+ int ne10,
237
+ int ne12,
238
+ int ne0,
239
+ int ne1,
240
+ int r2,
241
+ int r3
242
+ ) {
243
+ src1 = (global float*)((global char*)src1 + offset1);
244
+ dst = (global float*)((global char*)dst + offsetd);
245
+
246
+ mul_vec_q_n_f32_flat(src0_qs, src0_qh, src0_d, src0_m, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3);
247
+ }
llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32.cl ADDED
@@ -0,0 +1,187 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #pragma OPENCL EXTENSION cl_khr_fp16 : enable
2
+
3
+ #ifdef cl_intel_subgroups
4
+ #pragma OPENCL EXTENSION cl_intel_subgroups : enable
5
+ #else
6
+ #pragma OPENCL EXTENSION cl_khr_subgroups : enable
7
+ #endif
8
+
9
+ #ifdef cl_intel_required_subgroup_size
10
+ #pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
11
+ #define INTEL_GPU 1
12
+ #define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
13
+ #define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
14
+ #elif defined(cl_qcom_reqd_sub_group_size)
15
+ #pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
16
+ #define ADRENO_GPU 1
17
+ #define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
18
+ #define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
19
+ #endif
20
+
21
+ #define QK_K 256
22
+ #define K_SCALE_SIZE 12
23
+
24
+ typedef struct {
25
+ half d; // super-block scale for quantized scales
26
+ half dmin; // super-block scale for quantized mins
27
+ uchar scales[K_SCALE_SIZE]; // scales and mins, quantized with 6 bits
28
+ uchar qh[QK_K/8]; // quants, high bit (1 bit per value, packed 8 per byte)
29
+ uchar qs[QK_K/2]; // quants, low 4 bits (2 values per byte)
30
+ } block_q5_K;
31
+
32
+ #undef N_DST
33
+ #undef N_SIMDGROUP
34
+ #undef N_SIMDWIDTH
35
+
36
+ #ifdef INTEL_GPU
37
+ #define N_DST 4
38
+ #define N_SIMDGROUP 1
39
+ #define N_SIMDWIDTH 16
40
+ #elif defined(ADRENO_GPU)
41
+ #define N_DST 4
42
+ #define N_SIMDGROUP 1
43
+ #define N_SIMDWIDTH 64
44
+ #endif
45
+
46
+ #define BLOCK_STRIDE (N_SIMDWIDTH/8)
47
+
48
+ #ifdef INTEL_GPU
49
+ REQD_SUBGROUP_SIZE_16
50
+ #elif defined (ADRENO_GPU)
51
+ REQD_SUBGROUP_SIZE_64
52
+ #endif
53
+ kernel void kernel_mul_mv_q5_K_f32(
54
+ global char * src0,
55
+ int offset0,
56
+ global char * src1,
57
+ int offset1,
58
+ global char * dst,
59
+ int offsetd,
60
+ int ne00,
61
+ int ne01,
62
+ ulong nb01,
63
+ ulong nb02,
64
+ ulong nb03,
65
+ int ne12,
66
+ ulong nb11,
67
+ ulong nb12,
68
+ ulong nb13,
69
+ int ne0,
70
+ int ne1,
71
+ int r2,
72
+ int r3
73
+ ) {
74
+ src0 = src0 + offset0;
75
+ src1 = src1 + offset1;
76
+ dst = dst + offsetd;
77
+
78
+ ushort kmask1 = 0x3f3f;
79
+ ushort kmask2 = 0x0f0f;
80
+ ushort kmask3 = 0xc0c0;
81
+
82
+ int ix = get_sub_group_local_id()/8; // super block index
83
+ int it = get_sub_group_local_id()%8; // block index (inside super block)
84
+ int iq = it/4; // 0 or 1 - first or second half of the super block
85
+ int ir = it%4; // 0...3 - block index in the half super block
86
+
87
+ int nb = ne00/QK_K;
88
+
89
+ int r0 = get_group_id(0);
90
+ int r1 = get_group_id(1);
91
+ int im = get_group_id(2);
92
+ int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;
93
+
94
+ int i12 = im%ne12;
95
+ int i13 = im/ne12;
96
+
97
+ int offset_src0 = first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03;
98
+ int offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13;
99
+
100
+ global block_q5_K * x = (global block_q5_K *) (src0 + offset_src0);
101
+ global float * y = (global float *) (src1 + offset_src1);
102
+
103
+ float yl[16];
104
+ float yh[16];
105
+ float sumf[N_DST] = {0.f};
106
+ float all_sum;
107
+
108
+ global float * y4 = y + ix * QK_K + 64 * iq + 8 * ir;
109
+
110
+ uchar u1_lo = (uchar)(1 << (2*iq));
111
+ uchar u2_lo = (uchar)(2 << (2*iq));
112
+ uchar u1_hi = (uchar)(1 << (2*iq + 4));
113
+ uchar u2_hi = (uchar)(2 << (2*iq + 4));
114
+
115
+ ushort sc16[4];
116
+ uchar * sc8 = (uchar *)sc16;
117
+
118
+ for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) {
119
+ float4 sumy = {0.f, 0.f, 0.f, 0.f};
120
+ for (int i = 0; i < 8; ++i) {
121
+ yl[i+0] = y4[i+0];
122
+ sumy.s0 += yl[i+0];
123
+
124
+ yl[i+8] = y4[i+32];
125
+ sumy.s1 += yl[i+8];
126
+
127
+ yh[i+0] = y4[i+128];
128
+ sumy.s2 += yh[i+0];
129
+
130
+ yh[i+8] = y4[i+160];
131
+ sumy.s3 += yh[i+8];
132
+ }
133
+
134
+ global ushort * sc = (global ushort *)x[ib].scales + iq;
135
+ global ushort * q1 = (global ushort *)x[ib].qs + 16 * iq + 4 * ir;
136
+ global uchar * qh = x[ib].qh + 8 * ir;
137
+ global half * dh = &x[ib].d;
138
+
139
+ for (int row = 0; row < N_DST; row++) {
140
+ sc16[0] = sc[0] & kmask1;
141
+ sc16[1] = sc[2] & kmask1;
142
+ sc16[2] = ((sc[4] >> 0) & kmask2) | ((sc[0] & kmask3) >> 2);
143
+ sc16[3] = ((sc[4] >> 4) & kmask2) | ((sc[2] & kmask3) >> 2);
144
+
145
+ global ushort * q2 = q1 + 32;
146
+
147
+ float4 acc1 = {0.f, 0.f, 0.f, 0.f};
148
+ float4 acc2 = {0.f, 0.f, 0.f, 0.f};
149
+ for (int i = 0; i < 8; i += 2) {
150
+ acc1.s0 += yl[i+0] * ((q1[i/2] & 0x000F) + (qh[i+0] & u1_lo ? 16.f : 0.f));
151
+ acc1.s1 += yl[i+1] * ((q1[i/2] & 0x0F00) + (qh[i+1] & u1_lo ? 16.f*256.f : 0.f));
152
+ acc1.s2 += yl[i+8] * ((q1[i/2] & 0x00F0) + (qh[i+0] & u2_lo ? 16.f*16.f : 0.f));
153
+ acc1.s3 += yl[i+9] * ((q1[i/2] & 0xF000) + (qh[i+1] & u2_lo ? 16.f*4096.f: 0.f));
154
+ acc2.s0 += yh[i+0] * ((q2[i/2] & 0x000F) + (qh[i+0] & u1_hi ? 16.f : 0.f));
155
+ acc2.s1 += yh[i+1] * ((q2[i/2] & 0x0F00) + (qh[i+1] & u1_hi ? 16.f*256.f : 0.f));
156
+ acc2.s2 += yh[i+8] * ((q2[i/2] & 0x00F0) + (qh[i+0] & u2_hi ? 16.f*16.f : 0.f));
157
+ acc2.s3 += yh[i+9] * ((q2[i/2] & 0xF000) + (qh[i+1] & u2_hi ? 16.f*4096.f: 0.f));
158
+ }
159
+
160
+ float dall = dh[0];
161
+ float dmin = dh[1];
162
+ sumf[row] += dall * ((acc1.s0 + 1.f/256.f * acc1.s1) * sc8[0] +
163
+ (acc1.s2 + 1.f/256.f * acc1.s3) * sc8[1] * 1.f/16.f +
164
+ (acc2.s0 + 1.f/256.f * acc2.s1) * sc8[4] +
165
+ (acc2.s2 + 1.f/256.f * acc2.s3) * sc8[5] * 1.f/16.f) -
166
+ dmin * (sumy.s0 * sc8[2] + sumy.s1 * sc8[3] + sumy.s2 * sc8[6] + sumy.s3 * sc8[7]);
167
+
168
+ q1 += nb01/2;
169
+ sc += nb01/2;
170
+ dh += nb01/2;
171
+ qh += nb01;
172
+ }
173
+
174
+ y4 += BLOCK_STRIDE * QK_K;
175
+ }
176
+
177
+ global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0;
178
+
179
+ for (int row = 0; row < N_DST; ++row) {
180
+ all_sum = sub_group_reduce_add(sumf[row]);
181
+ if (first_row + row < ne01) {
182
+ if (get_sub_group_local_id() == 0) {
183
+ dst_f32[first_row + row] = all_sum;
184
+ }
185
+ }
186
+ }
187
+ }
llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32_flat.cl ADDED
@@ -0,0 +1,203 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #pragma OPENCL EXTENSION cl_khr_fp16 : enable
2
+
3
+ #ifdef cl_intel_subgroups
4
+ #pragma OPENCL EXTENSION cl_intel_subgroups : enable
5
+ #else
6
+ #pragma OPENCL EXTENSION cl_khr_subgroups : enable
7
+ #endif
8
+
9
+ #ifdef cl_intel_required_subgroup_size
10
+ #pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
11
+ #define INTEL_GPU 1
12
+ #define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
13
+ #define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
14
+ #elif defined(cl_qcom_reqd_sub_group_size)
15
+ #pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
16
+ #define ADRENO_GPU 1
17
+ #define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
18
+ #define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
19
+ #endif
20
+
21
+ //------------------------------------------------------------------------------
22
+ // block_q5_K
23
+ //------------------------------------------------------------------------------
24
+ #define QK_K 256
25
+ #define BLOCK_Q5K_SIZE 176
26
+ #define K_SCALE_SIZE 12
27
+
28
+ typedef struct {
29
+ half d; // super-block scale for quantized scales
30
+ half dmin; // super-block scale for quantized mins
31
+ uchar scales[K_SCALE_SIZE]; // scales and mins, quantized with 6 bits
32
+ uchar qh[QK_K/8]; // quants, high bit (1 bit per value, packed 8 per byte)
33
+ uchar qs[QK_K/2]; // quants, low 4 bits (2 values per byte)
34
+ } block_q5_K;
35
+
36
+ #undef N_DST
37
+ #undef N_SIMDGROUP
38
+ #undef N_SIMDWIDTH
39
+
40
+ #ifdef INTEL_GPU
41
+ #define N_DST 4
42
+ #define N_SIMDGROUP 1
43
+ #define N_SIMDWIDTH 16
44
+ #elif defined(ADRENO_GPU)
45
+ #define N_DST 16
46
+ #define N_SIMDGROUP 2
47
+ #define N_SIMDWIDTH 64
48
+ #endif
49
+
50
+ #undef BLOCK_STRIDE
51
+ // number of (super) blocks each subgroup processes
52
+ // each thread in a subgroup processes a block (32 weights)
53
+ #define BLOCK_STRIDE (N_SIMDWIDTH/8)
54
+
55
+ #ifdef INTEL_GPU
56
+ REQD_SUBGROUP_SIZE_16
57
+ #elif defined (ADRENO_GPU)
58
+ REQD_SUBGROUP_SIZE_64
59
+ #endif
60
+ kernel void kernel_mul_mv_q5_K_f32_flat(
61
+ global uchar * src0_q,
62
+ global uchar * src0_qh,
63
+ global uchar * src0_s,
64
+ global half * src0_d,
65
+ global half * src0_dm,
66
+ global char * src1,
67
+ int offset1,
68
+ global char * dst,
69
+ int offsetd,
70
+ int ne00,
71
+ int ne01,
72
+ ulong nb01,
73
+ ulong nb02,
74
+ ulong nb03,
75
+ int ne12,
76
+ ulong nb11,
77
+ ulong nb12,
78
+ ulong nb13,
79
+ int ne0,
80
+ int ne1,
81
+ int r2,
82
+ int r3
83
+ ) {
84
+ src1 = src1 + offset1;
85
+ dst = dst + offsetd;
86
+
87
+ ushort kmask1 = 0x3f3f;
88
+ ushort kmask2 = 0x0f0f;
89
+ ushort kmask3 = 0xc0c0;
90
+
91
+ int ix = get_sub_group_local_id()/8;
92
+ int it = get_sub_group_local_id()%8;
93
+ int iq = it/4;
94
+ int ir = it%4;
95
+
96
+ int nb = ne00/QK_K;
97
+
98
+ int r0 = get_group_id(0);
99
+ int r1 = get_group_id(1);
100
+ int im = get_group_id(2);
101
+ int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;
102
+
103
+ int i12 = im%ne12;
104
+ int i13 = im/ne12;
105
+
106
+ int offset_src0 = (first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03)/BLOCK_Q5K_SIZE;
107
+ uint blk = nb01 / BLOCK_Q5K_SIZE;
108
+ global uchar * blk_q = (global uchar *)src0_q + offset_src0*(QK_K/2);
109
+ global uchar * blk_qh = (global uchar *)src0_qh + offset_src0*(QK_K/8);
110
+ global uchar * blk_s = (global uchar *)src0_s + offset_src0*K_SCALE_SIZE;
111
+ global half * blk_d = (global half *)src0_d + offset_src0;
112
+ global half * blk_dm = (global half *)src0_dm + offset_src0;
113
+
114
+ int offset_src1 = r1*nb11 + (i12)*nb12 + (i13)*nb13;
115
+ global float * y = (global float *)(src1 + offset_src1);
116
+
117
+ float yl[16];
118
+ float yh[16];
119
+ float sumf[N_DST] = {0.f};
120
+ float all_sum;
121
+
122
+ global float * y4 = y + ix * QK_K + 64 * iq + 8 * ir;
123
+
124
+ uchar u1_lo = (uchar)(1 << (2*iq));
125
+ uchar u2_lo = (uchar)(2 << (2*iq));
126
+ uchar u1_hi = (uchar)(1 << (2*iq + 4));
127
+ uchar u2_hi = (uchar)(2 << (2*iq + 4));
128
+
129
+ ushort sc16[4];
130
+ uchar * sc8 = (uchar *)sc16;
131
+
132
+ for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) {
133
+ float4 sumy = {0.f, 0.f, 0.f, 0.f};
134
+ for (int i = 0; i < 8; ++i) {
135
+ yl[i+0] = y4[i+0];
136
+ sumy.s0 += yl[i+0];
137
+
138
+ yl[i+8] = y4[i+32];
139
+ sumy.s1 += yl[i+8];
140
+
141
+ yh[i+0] = y4[i+128];
142
+ sumy.s2 += yh[i+0];
143
+
144
+ yh[i+8] = y4[i+160];
145
+ sumy.s3 += yh[i+8];
146
+ }
147
+
148
+ global ushort * q1 = (global ushort *)(blk_q + ib * (QK_K/2)) + (16 * iq + 4 * ir);
149
+ global uchar * qh = (global uchar *)(blk_qh + ib * (QK_K/8)) + 8 * ir;
150
+ global ushort * sc = (global ushort *)(blk_s + ib * K_SCALE_SIZE) + iq;
151
+ global half * d = blk_d + ib;
152
+ global half * dm = blk_dm + ib;
153
+
154
+ for (int row = 0; row < N_DST; row++) {
155
+ sc16[0] = sc[0] & kmask1;
156
+ sc16[1] = sc[2] & kmask1;
157
+ sc16[2] = ((sc[4] >> 0) & kmask2) | ((sc[0] & kmask3) >> 2);
158
+ sc16[3] = ((sc[4] >> 4) & kmask2) | ((sc[2] & kmask3) >> 2);
159
+
160
+ global ushort * q2 = q1 + 32;
161
+
162
+ float4 acc1 = {0.f, 0.f, 0.f, 0.f};
163
+ float4 acc2 = {0.f, 0.f, 0.f, 0.f};
164
+ for (int i = 0; i < 8; i += 2) {
165
+ acc1.s0 += yl[i+0] * ((q1[i/2] & 0x000F) + (qh[i+0] & u1_lo ? 16.f : 0.f));
166
+ acc1.s1 += yl[i+1] * ((q1[i/2] & 0x0F00) + (qh[i+1] & u1_lo ? 16.f*256.f : 0.f));
167
+ acc1.s2 += yl[i+8] * ((q1[i/2] & 0x00F0) + (qh[i+0] & u2_lo ? 16.f*16.f : 0.f));
168
+ acc1.s3 += yl[i+9] * ((q1[i/2] & 0xF000) + (qh[i+1] & u2_lo ? 16.f*4096.f: 0.f));
169
+ acc2.s0 += yh[i+0] * ((q2[i/2] & 0x000F) + (qh[i+0] & u1_hi ? 16.f : 0.f));
170
+ acc2.s1 += yh[i+1] * ((q2[i/2] & 0x0F00) + (qh[i+1] & u1_hi ? 16.f*256.f : 0.f));
171
+ acc2.s2 += yh[i+8] * ((q2[i/2] & 0x00F0) + (qh[i+0] & u2_hi ? 16.f*16.f : 0.f));
172
+ acc2.s3 += yh[i+9] * ((q2[i/2] & 0xF000) + (qh[i+1] & u2_hi ? 16.f*4096.f: 0.f));
173
+ }
174
+
175
+ float dall = *d;
176
+ float dmin = *dm;
177
+ sumf[row] += dall * ((acc1.s0 + 1.f/256.f * acc1.s1) * sc8[0] +
178
+ (acc1.s2 + 1.f/256.f * acc1.s3) * sc8[1] * 1.f/16.f +
179
+ (acc2.s0 + 1.f/256.f * acc2.s1) * sc8[4] +
180
+ (acc2.s2 + 1.f/256.f * acc2.s3) * sc8[5] * 1.f/16.f) -
181
+ dmin * (sumy.s0 * sc8[2] + sumy.s1 * sc8[3] + sumy.s2 * sc8[6] + sumy.s3 * sc8[7]);
182
+
183
+ q1 += blk*64;
184
+ qh += blk*32;
185
+ sc += blk*6;
186
+ d += blk;
187
+ dm += blk;
188
+ }
189
+
190
+ y4 += BLOCK_STRIDE * QK_K;
191
+ }
192
+
193
+ global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0;
194
+
195
+ for (int row = 0; row < N_DST; ++row) {
196
+ all_sum = sub_group_reduce_add(sumf[row]);
197
+ if (first_row + row < ne01) {
198
+ if (get_sub_group_local_id() == 0) {
199
+ dst_f32[first_row + row] = all_sum;
200
+ }
201
+ }
202
+ }
203
+ }
llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32.cl ADDED
@@ -0,0 +1,194 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #pragma OPENCL EXTENSION cl_khr_fp16 : enable
2
+
3
+ #ifdef cl_intel_subgroups
4
+ #pragma OPENCL EXTENSION cl_intel_subgroups : enable
5
+ #else
6
+ #pragma OPENCL EXTENSION cl_khr_subgroups : enable
7
+ #endif
8
+
9
+ #ifdef cl_intel_required_subgroup_size
10
+ #pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
11
+ #define INTEL_GPU 1
12
+ #define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
13
+ #define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
14
+ #elif defined(cl_qcom_reqd_sub_group_size)
15
+ #pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
16
+ #define ADRENO_GPU 1
17
+ #define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
18
+ #define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
19
+ #endif
20
+
21
+ #define QK4_0 32
22
+ #define QR4_0 2
23
+ #define QK4_1 32
24
+ #define QR4_1 2
25
+ #define QK5_0 32
26
+ #define QR5_0 2
27
+ #define QK5_1 32
28
+ #define QR5_1 2
29
+ #define QK8_0 32
30
+ #define QR8_0 1
31
+ #define QK_K 256
32
+ #define K_QUANTS_PER_ITERATION 2
33
+
34
+ typedef char int8_t;
35
+ typedef uchar uint8_t;
36
+ typedef short int16_t;
37
+ typedef ushort uint16_t;
38
+ typedef int int32_t;
39
+ typedef uint uint32_t;
40
+
41
+ //------------------------------------------------------------------------------
42
+ // block_q6_K
43
+ //------------------------------------------------------------------------------
44
+ // 6-bit quantization
45
+ // weight is represented as x = a * q
46
+ // 16 blocks of 16 elements each
47
+ // Effectively 6.5625 bits per weight
48
+ typedef struct {
49
+ uint8_t ql[QK_K/2]; // quants, lower 4 bits
50
+ uint8_t qh[QK_K/4]; // quants, upper 2 bits
51
+ int8_t scales[QK_K/16]; // scales, quantized with 8 bits
52
+ half d; // super-block scale
53
+ } block_q6_K;
54
+
55
+ //------------------------------------------------------------------------------
56
+ // kernel_mul_mv_q6_K_f32
57
+ //------------------------------------------------------------------------------
58
+
59
+ #undef N_DST
60
+ #undef N_SIMDGROUP
61
+ #undef N_SIMDWIDTH
62
+
63
+ #ifdef INTEL_GPU
64
+ #define N_DST 1 // number of rows each SIMD group works on
65
+ #define N_SIMDGROUP 2 // number of SIMD groups in a thread group
66
+ #define N_SIMDWIDTH 16 // SIMD group size
67
+ #elif defined (ADRENO_GPU)
68
+ #define N_DST 1
69
+ #define N_SIMDGROUP 2
70
+ #define N_SIMDWIDTH 64
71
+ #endif
72
+
73
+ #define BLOCK_STRIDE (N_SIMDWIDTH/16) // number of blocks each subgroup processes
74
+
75
+ #ifdef INTEL_GPU
76
+ REQD_SUBGROUP_SIZE_16
77
+ #elif defined (ADRENO_GPU)
78
+ REQD_SUBGROUP_SIZE_64
79
+ #endif
80
+ kernel void kernel_mul_mv_q6_K_f32(
81
+ global void * src0,
82
+ ulong offset0,
83
+ global float * src1,
84
+ ulong offset1,
85
+ global float * dst,
86
+ ulong offsetd,
87
+ int ne00,
88
+ int ne01,
89
+ int ne02,
90
+ int ne10,
91
+ int ne12,
92
+ int ne0,
93
+ int ne1,
94
+ int r2,
95
+ int r3
96
+ ) {
97
+ src0 = (global void*)((global char*)src0 + offset0);
98
+ src1 = (global float*)((global char*)src1 + offset1);
99
+ dst = (global float*)((global char*)dst + offsetd);
100
+
101
+ uchar kmask1 = 0x03;
102
+ uchar kmask2 = 0x0C;
103
+ uchar kmask3 = 0x30;
104
+ uchar kmask4 = 0xC0;
105
+
106
+ int nb = ne00/QK_K;
107
+
108
+ int r0 = get_group_id(0);
109
+ int r1 = get_group_id(1);
110
+ int im = get_group_id(2);
111
+
112
+ int row = N_SIMDGROUP * r0 + get_sub_group_id();
113
+
114
+ if (row >= ne01) {
115
+ return;
116
+ }
117
+
118
+ int i12 = im%ne12;
119
+ int i13 = im/ne12;
120
+
121
+ ulong offset_src0 = (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);
122
+
123
+ global block_q6_K * x = (global block_q6_K *) src0 + row*nb + offset_src0;
124
+ global float * yy = (global float *) src1 + r1*ne10 + im*ne00*ne1;
125
+
126
+ float sumf = 0;
127
+
128
+ // For Q6_K quantization, 16 values forms a subblock, 16 subblock forms a
129
+ // block. Values in a subblock shares a scale that is quantized with 8 bits;
130
+ // the entire block shares a single floating point scale.
131
+ // For work distribution, each thread processes a subblock (16 weights), hence
132
+ // 16 threads process a (super) block -- a subgroup thus handles SIMDWIDTH/16
133
+ // (super) blocks -- this is the block stride.
134
+ // The 16 threads that process a (super) block are split into 2 portions, each has
135
+ // 8 threads; each portion works on 8 subblocks.
136
+ // For subgroup of 16 threads, the entire subgroup works on a single (super) block
137
+ // before moving to the next (super) block. Thread0 - thread7 work on the
138
+ // first 8 subblocks; thread8 - thread15 works on the last 8 subblocks.
139
+ // Thread0 - thread3 work on subblocks 0, 2, 4, 6; thread4 - thread7 work on
140
+ // subblocks 1, 3, 5, 7. Each thread does not work on an entire subblock, but
141
+ // works on a total of 16 weight values.
142
+ int tid = get_sub_group_local_id()/BLOCK_STRIDE; // first block_stride groups have tid=0
143
+ int ix = get_sub_group_local_id()%BLOCK_STRIDE; // first block is 0..block_stride-1
144
+ int ip = tid/8; // first or second half of (super) block (0 or 1)
145
+ int il = tid%8; // each half has 8 parts, one per scale
146
+ int n = 4; // 4 scales at a time (and 4 sums)
147
+ int l0 = n*il; // offset into half-block, 0..28
148
+ int is = 8*ip + l0/16; // 0, 1, 8, 9
149
+
150
+ int y_offset = 128*ip + l0;
151
+ int q_offset_l = 64*ip + l0;
152
+ int q_offset_h = 32*ip + l0;
153
+
154
+ for (int i = ix; i < nb; i += BLOCK_STRIDE) {
155
+
156
+ global uint8_t * q1 = x[i].ql + q_offset_l;
157
+ global uint8_t * q2 = q1 + QK_K/8;
158
+ global uint8_t * qh = x[i].qh + q_offset_h;
159
+ global int8_t * sc = x[i].scales + is;
160
+
161
+ global float * y = yy + i * QK_K + y_offset;
162
+
163
+ float dall = x[i].d;
164
+
165
+ float4 sums = {0.f, 0.f, 0.f, 0.f};
166
+
167
+ sums.s0 += y[0+ 0] * ((float)((q1[0] & 0xF) | ((qh[0] & kmask1) << 4)) - 32.f);
168
+ sums.s1 += y[0+32] * ((float)((q2[0] & 0xF) | ((qh[0] & kmask2) << 2)) - 32.f);
169
+ sums.s2 += y[0+64] * ((float)((q1[0] >> 4) | ((qh[0] & kmask3) << 0)) - 32.f);
170
+ sums.s3 += y[0+96] * ((float)((q2[0] >> 4) | ((qh[0] & kmask4) >> 2)) - 32.f);
171
+
172
+ sums.s0 += y[1+ 0] * ((float)((q1[1] & 0xF) | ((qh[1] & kmask1) << 4)) - 32.f);
173
+ sums.s1 += y[1+32] * ((float)((q2[1] & 0xF) | ((qh[1] & kmask2) << 2)) - 32.f);
174
+ sums.s2 += y[1+64] * ((float)((q1[1] >> 4) | ((qh[1] & kmask3) << 0)) - 32.f);
175
+ sums.s3 += y[1+96] * ((float)((q2[1] >> 4) | ((qh[1] & kmask4) >> 2)) - 32.f);
176
+
177
+ sums.s0 += y[2+ 0] * ((float)((q1[2] & 0xF) | ((qh[2] & kmask1) << 4)) - 32.f);
178
+ sums.s1 += y[2+32] * ((float)((q2[2] & 0xF) | ((qh[2] & kmask2) << 2)) - 32.f);
179
+ sums.s2 += y[2+64] * ((float)((q1[2] >> 4) | ((qh[2] & kmask3) << 0)) - 32.f);
180
+ sums.s3 += y[2+96] * ((float)((q2[2] >> 4) | ((qh[2] & kmask4) >> 2)) - 32.f);
181
+
182
+ sums.s0 += y[3+ 0] * ((float)((q1[3] & 0xF) | ((qh[3] & kmask1) << 4)) - 32.f);
183
+ sums.s1 += y[3+32] * ((float)((q2[3] & 0xF) | ((qh[3] & kmask2) << 2)) - 32.f);
184
+ sums.s2 += y[3+64] * ((float)((q1[3] >> 4) | ((qh[3] & kmask3) << 0)) - 32.f);
185
+ sums.s3 += y[3+96] * ((float)((q2[3] >> 4) | ((qh[3] & kmask4) >> 2)) - 32.f);
186
+
187
+ sumf += dall * (sums.s0 * sc[0] + sums.s1 * sc[2] + sums.s2 * sc[4] + sums.s3 * sc[6]);
188
+ }
189
+
190
+ float tot = sub_group_reduce_add(sumf);
191
+ if (get_sub_group_local_id() == 0) {
192
+ dst[r1*ne0 + im*ne0*ne1 + row] = tot;
193
+ }
194
+ }
llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32_flat.cl ADDED
@@ -0,0 +1,178 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #pragma OPENCL EXTENSION cl_khr_fp16 : enable
2
+
3
+ #ifdef cl_intel_subgroups
4
+ #pragma OPENCL EXTENSION cl_intel_subgroups : enable
5
+ #else
6
+ #pragma OPENCL EXTENSION cl_khr_subgroups : enable
7
+ #endif
8
+
9
+ #ifdef cl_intel_required_subgroup_size
10
+ #pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
11
+ #define INTEL_GPU 1
12
+ #define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
13
+ #define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
14
+ #elif defined(cl_qcom_reqd_sub_group_size)
15
+ #pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
16
+ #define ADRENO_GPU 1
17
+ #define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
18
+ #define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
19
+ #endif
20
+
21
+ //------------------------------------------------------------------------------
22
+ // kernel_mul_mv_q6_K_f32_flat
23
+ //------------------------------------------------------------------------------
24
+ #define Q6_K_MASK1 0x03
25
+ #define Q6_K_MASK2 0x0C
26
+ #define Q6_K_MASK3 0x30
27
+ #define Q6_K_MASK4 0xC0
28
+
29
+ #define QK_K 256
30
+
31
+ inline float block_q_6_K_dot_y_flat(
32
+ global uchar * blk_ql,
33
+ global uchar * blk_qh,
34
+ global char * blk_scales,
35
+ global half * blk_d,
36
+ int ib,
37
+ int ip,
38
+ int is,
39
+ int l0,
40
+ float4 y0,
41
+ float4 y1,
42
+ float4 y2,
43
+ float4 y3
44
+ ) {
45
+ int q_offset_l = 64*ip + l0;
46
+ int q_offset_h = 32*ip + l0;
47
+
48
+ global uchar * q1 = blk_ql + ib*128 + q_offset_l;
49
+ global uchar * q2 = q1 + QK_K/8;
50
+ global uchar * qh = blk_qh + ib*64 + q_offset_h;
51
+ global char * sc = blk_scales + ib*16 + is;
52
+
53
+ float dall = blk_d[ib];
54
+
55
+ // Vectorized loads: 3 uchar4 weight loads instead of 12 scalar byte reads.
56
+ // q_offset_l/h are 4-aligned, so these are aligned vector loads.
57
+ uchar4 q1v = vload4(0, q1);
58
+ uchar4 q2v = vload4(0, q2);
59
+ uchar4 qhv = vload4(0, qh);
60
+
61
+ int4 q1i = convert_int4(q1v);
62
+ int4 q2i = convert_int4(q2v);
63
+ int4 qhi = convert_int4(qhv);
64
+
65
+ // Reconstruct the four 6-bit weight groups (low/high nibble of ql OR'd with the
66
+ // matching 2-bit plane of qh), same arithmetic as the scalar version, then dot()
67
+ // against the cached activation lanes.
68
+ float4 w0 = convert_float4((q1i & 0xF) | ((qhi & Q6_K_MASK1) << 4)) - 32.f;
69
+ float4 w1 = convert_float4((q2i & 0xF) | ((qhi & Q6_K_MASK2) << 2)) - 32.f;
70
+ float4 w2 = convert_float4((q1i >> 4) | ((qhi & Q6_K_MASK3) )) - 32.f;
71
+ float4 w3 = convert_float4((q2i >> 4) | ((qhi & Q6_K_MASK4) >> 2)) - 32.f;
72
+
73
+ return dall * (dot(y0, w0) * sc[0] + dot(y1, w1) * sc[2] +
74
+ dot(y2, w2) * sc[4] + dot(y3, w3) * sc[6]);
75
+ }
76
+
77
+ #undef N_DST
78
+ #undef N_SIMDGROUP
79
+ #undef N_SIMDWIDTH
80
+
81
+ #ifdef INTEL_GPU
82
+ #define N_DST 4
83
+ #define N_SIMDGROUP 2
84
+ #define N_SIMDWIDTH 16
85
+ #elif defined (ADRENO_GPU)
86
+ #define N_DST 16
87
+ #define N_SIMDGROUP 2
88
+ #define N_SIMDWIDTH 64
89
+ #endif
90
+
91
+ #define BLOCK_STRIDE (N_SIMDWIDTH/16) // number of blocks each subgroup processes
92
+
93
+ #ifdef INTEL_GPU
94
+ REQD_SUBGROUP_SIZE_16
95
+ #elif defined (ADRENO_GPU)
96
+ REQD_SUBGROUP_SIZE_64
97
+ #endif
98
+ kernel void kernel_mul_mv_q6_K_f32_flat(
99
+ global uchar * src0_ql,
100
+ global uchar * src0_qh,
101
+ global char * src0_s,
102
+ global half * src0_d,
103
+ global float * src1,
104
+ ulong offset1,
105
+ global float * dst,
106
+ ulong offsetd,
107
+ int ne00,
108
+ int ne01,
109
+ int ne02,
110
+ int ne10,
111
+ int ne12,
112
+ int ne0,
113
+ int ne1,
114
+ int r2,
115
+ int r3
116
+ ) {
117
+ src1 = (global float*)((global char*)src1 + offset1);
118
+ dst = (global float*)((global char*)dst + offsetd);
119
+
120
+ int nb = ne00/QK_K;
121
+
122
+ int r0 = get_group_id(0);
123
+ int r1 = get_group_id(1);
124
+ int im = get_group_id(2);
125
+
126
+ int i12 = im%ne12;
127
+ int i13 = im/ne12;
128
+
129
+ int first_row = (N_SIMDGROUP * r0 + get_sub_group_id()) * N_DST;
130
+
131
+ ulong offset_src0 = first_row*nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);
132
+ ulong offset_src0_ql = offset_src0 * 128;
133
+ ulong offset_src0_qh = offset_src0 * 64;
134
+ ulong offset_src0_s = offset_src0 * 16;
135
+ ulong offset_src0_d = offset_src0;
136
+
137
+ global uchar * blk_ql = (global uchar *) src0_ql + offset_src0_ql;
138
+ global uchar * blk_qh = (global uchar *) src0_qh + offset_src0_qh;
139
+ global char * blk_scales = (global char *) src0_s + offset_src0_s;
140
+ global half * blk_d = (global half *) src0_d + offset_src0_d;
141
+ global float * yy = (global float *) src1 + r1*ne10 + im*ne00*ne1;
142
+
143
+ int tid = get_sub_group_local_id()%(N_SIMDWIDTH/BLOCK_STRIDE); // within-super-block part, 0..15
144
+ int ix = get_sub_group_local_id()/(N_SIMDWIDTH/BLOCK_STRIDE); // super-block selector, 0..BLOCK_STRIDE-1
145
+ int ip = tid/8; // first or second half of (super) block (0 or 1)
146
+ int il = tid%8; // each half has 8 parts, one per scale
147
+ int n = 4; // 4 scales at a time (and 4 sums)
148
+ int l0 = n*il; // offset into half-block, 0..28
149
+ int is = 8*ip + l0/16; // 0, 1, 8, 9
150
+
151
+ float sumf[N_DST];
152
+ for (int row = 0; row < N_DST; row++) {
153
+ sumf[row] = 0.f;
154
+ }
155
+
156
+ for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) {
157
+ global float * y = yy + ib * QK_K + 128*ip + l0;
158
+ float4 y0 = vload4(0, y + 0);
159
+ float4 y1 = vload4(0, y + 32);
160
+ float4 y2 = vload4(0, y + 64);
161
+ float4 y3 = vload4(0, y + 96);
162
+
163
+ for (int row = 0; row < N_DST; row++) {
164
+ if (first_row + row < ne01) {
165
+ sumf[row] += block_q_6_K_dot_y_flat(
166
+ blk_ql + row*nb*128, blk_qh + row*nb*64, blk_scales + row*nb*16, blk_d + row*nb,
167
+ ib, ip, is, l0, y0, y1, y2, y3);
168
+ }
169
+ }
170
+ }
171
+
172
+ for (int row = 0; row < N_DST; row++) {
173
+ float tot = sub_group_reduce_add(sumf[row]);
174
+ if (get_sub_group_local_id() == 0 && first_row + row < ne01) {
175
+ dst[r1*ne0 + im*ne0*ne1 + first_row + row] = tot;
176
+ }
177
+ }
178
+ }
llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_q8_0_f32.cl ADDED
@@ -0,0 +1,125 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #pragma OPENCL EXTENSION cl_khr_fp16 : enable
2
+
3
+ #ifdef cl_intel_subgroups
4
+ #pragma OPENCL EXTENSION cl_intel_subgroups : enable
5
+ #else
6
+ #pragma OPENCL EXTENSION cl_khr_subgroups : enable
7
+ #endif
8
+
9
+ #ifdef cl_intel_required_subgroup_size
10
+ #pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
11
+ #define INTEL_GPU 1
12
+ #define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
13
+ #define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
14
+ #elif defined(cl_qcom_reqd_sub_group_size)
15
+ #pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
16
+ #define ADRENO_GPU 1
17
+ #define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
18
+ #define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
19
+ #endif
20
+
21
+ #define QK8_0 32
22
+ typedef struct {
23
+ half d; // delta
24
+ char qs[QK8_0]; // quants
25
+ } block_q8_0;
26
+
27
+ #define NB_Q8_0 8
28
+
29
+ #ifdef INTEL_GPU
30
+ #define N_R0_Q8_0 4 // number of rows each subgroup works on
31
+ #define N_SG_Q8_0 2 // number of subgroups in a work group
32
+ #define N_SIMDWIDTH 16 // subgroup size
33
+ #elif defined (ADRENO_GPU)
34
+ #define N_R0_Q8_0 4
35
+ #define N_SG_Q8_0 2
36
+ #define N_SIMDWIDTH 64
37
+ #endif
38
+
39
+ #ifdef INTEL_GPU
40
+ REQD_SUBGROUP_SIZE_16
41
+ #elif defined (ADRENO_GPU)
42
+ REQD_SUBGROUP_SIZE_64
43
+ #endif
44
+ kernel void kernel_mul_mv_q8_0_f32(
45
+ global char * src0,
46
+ ulong offset0,
47
+ global char * src1,
48
+ ulong offset1,
49
+ global char * dst,
50
+ ulong offsetd,
51
+ int ne00,
52
+ int ne01,
53
+ ulong nb01,
54
+ ulong nb02,
55
+ ulong nb03,
56
+ int ne12,
57
+ ulong nb11,
58
+ ulong nb12,
59
+ ulong nb13,
60
+ int ne0,
61
+ int ne1,
62
+ int r2,
63
+ int r3
64
+ ) {
65
+ src0 = (global char*)((global char*)src0 + offset0);
66
+ src1 = (global char*)((global char*)src1 + offset1);
67
+ dst = (global char*)((global char*)dst + offsetd);
68
+
69
+ int nb = ne00/QK8_0;
70
+
71
+ int r0 = get_group_id(0);
72
+ int r1 = get_group_id(1);
73
+ int im = get_group_id(2);
74
+
75
+ int first_row = (r0*N_SG_Q8_0 + get_sub_group_id()) * N_R0_Q8_0;
76
+
77
+ uint i12 = im%ne12;
78
+ uint i13 = im/ne12;
79
+
80
+ ulong offset_src1 = r1*nb11 + i12*nb12 + i13*nb13;
81
+ global float * y = (global float *) (src1 + offset_src1);
82
+
83
+ // pointers to src0 rows
84
+ global block_q8_0 * ax[N_R0_Q8_0];
85
+ for (int row = 0; row < N_R0_Q8_0; ++row) {
86
+ ulong offset_src0 = (first_row + row)*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03;
87
+ ax[row] = (global block_q8_0 *) ((global char *) src0 + offset_src0);
88
+ }
89
+
90
+ float yl[NB_Q8_0];
91
+ float sumf[N_R0_Q8_0] = { 0.f };
92
+
93
+ const short ix = get_sub_group_local_id()/4;
94
+ const short il = get_sub_group_local_id()%4;
95
+
96
+ global float * yb = y + ix*QK8_0 + il*NB_Q8_0;
97
+
98
+ // each thread handles NB_Q8_0 quants at a time
99
+ for (int ib = ix; ib < nb; ib += N_SIMDWIDTH/4) {
100
+ for (short i = 0; i < NB_Q8_0; ++i) {
101
+ yl[i] = yb[i];
102
+ }
103
+
104
+ for (short row = 0; row < N_R0_Q8_0; row++) {
105
+ global char * qs = ax[row][ib].qs + il*NB_Q8_0;
106
+ float sumq = 0.f;
107
+ for (short iq = 0; iq < NB_Q8_0; ++iq) {
108
+ sumq += qs[iq] * yl[iq];
109
+ }
110
+ sumf[row] += sumq*ax[row][ib].d;
111
+ }
112
+
113
+ yb += N_SIMDWIDTH*NB_Q8_0;
114
+ }
115
+
116
+ global float * dst_f32 = (global float *) dst + (ulong)im*ne0*ne1 + (ulong)r1*ne0;
117
+
118
+ for (int row = 0; row < N_R0_Q8_0; ++row) {
119
+ float tot = sub_group_reduce_add(sumf[row]);
120
+
121
+ if (get_sub_group_local_id() == 0 && first_row + row < ne01) {
122
+ dst_f32[first_row + row] = tot;
123
+ }
124
+ }
125
+ }
llama.cpp/ggml/src/ggml-opencl/kernels/mul_mv_q8_0_f32_flat.cl ADDED
@@ -0,0 +1,202 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #pragma OPENCL EXTENSION cl_khr_fp16 : enable
2
+
3
+ #ifdef cl_intel_subgroups
4
+ #pragma OPENCL EXTENSION cl_intel_subgroups : enable
5
+ #else
6
+ #pragma OPENCL EXTENSION cl_khr_subgroups : enable
7
+ #endif
8
+
9
+ #ifdef cl_intel_required_subgroup_size
10
+ #pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
11
+ #define INTEL_GPU 1
12
+ #define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
13
+ #define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
14
+ #elif defined(cl_qcom_reqd_sub_group_size)
15
+ #pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
16
+ #define ADRENO_GPU 1
17
+ #define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
18
+ #define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
19
+ #endif
20
+
21
+ #define QK8_0 32
22
+ typedef struct {
23
+ half d; // delta
24
+ char qs[QK8_0]; // quants
25
+ } block_q8_0;
26
+
27
+ #define NB_Q8_0 8
28
+
29
+ #ifdef INTEL_GPU
30
+ #define N_R0_Q8_0 4 // number of rows each subgroup works on
31
+ #define N_SG_Q8_0 2 // number of subgroups in a work group
32
+ #define N_SIMDWIDTH 16 // subgroup size
33
+ #elif defined (ADRENO_GPU)
34
+ #define N_R0_Q8_0 4
35
+ #define N_SG_Q8_0 2
36
+ #define N_SIMDWIDTH 64
37
+ #endif
38
+
39
+ #ifdef INTEL_GPU
40
+ REQD_SUBGROUP_SIZE_16
41
+ #elif defined (ADRENO_GPU)
42
+ REQD_SUBGROUP_SIZE_64
43
+ #endif
44
+ kernel void kernel_mul_mv_q8_0_f32_flat(
45
+ global char * src0_q,
46
+ global half * src0_d,
47
+ global char * src1,
48
+ ulong offset1,
49
+ global char * dst,
50
+ ulong offsetd,
51
+ int ne00,
52
+ int ne01,
53
+ ulong nb01,
54
+ ulong nb02,
55
+ ulong nb03,
56
+ int ne12,
57
+ ulong nb11,
58
+ ulong nb12,
59
+ ulong nb13,
60
+ int ne0,
61
+ int ne1,
62
+ int r2,
63
+ int r3
64
+ ) {
65
+ src1 = (global char*)((global char*)src1 + offset1);
66
+ dst = (global char*)((global char*)dst + offsetd);
67
+
68
+ int nb = ne00/QK8_0;
69
+
70
+ int r0 = get_group_id(0);
71
+ int r1 = get_group_id(1);
72
+ int im = get_group_id(2);
73
+
74
+ int first_row = (r0*N_SG_Q8_0 + get_sub_group_id()) * N_R0_Q8_0;
75
+
76
+ uint i12 = im%ne12;
77
+ uint i13 = im/ne12;
78
+
79
+ ulong offset_src1 = r1*nb11 + i12*nb12 + i13*nb13;
80
+ global float * y = (global float *) (src1 + offset_src1);
81
+
82
+ // pointers to src0 rows
83
+ uint offset_src0_base = first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03;
84
+
85
+ global char * ax0, * ax1, * ax2, * ax3;
86
+ global half * ad0, * ad1, * ad2, * ad3;
87
+ uint offset_src0;
88
+
89
+ offset_src0 = offset_src0_base + 0*nb01;
90
+ offset_src0 = offset_src0/34;
91
+ ax0 = (global char *) ((global char *) src0_q + offset_src0*sizeof(char)*QK8_0);
92
+ ad0 = (global half *) ((global char *) src0_d + offset_src0*sizeof(half));
93
+
94
+ offset_src0 = offset_src0_base + 1*nb01;
95
+ offset_src0 = offset_src0/34;
96
+ ax1 = (global char *) ((global char *) src0_q + offset_src0*sizeof(char)*QK8_0);
97
+ ad1 = (global half *) ((global char *) src0_d + offset_src0*sizeof(half));
98
+
99
+ offset_src0 = offset_src0_base + 2*nb01;
100
+ offset_src0 = offset_src0/34;
101
+ ax2 = (global char *) ((global char *) src0_q + offset_src0*sizeof(char)*QK8_0);
102
+ ad2 = (global half *) ((global char *) src0_d + offset_src0*sizeof(half));
103
+
104
+ offset_src0 = offset_src0_base + 3*nb01;
105
+ offset_src0 = offset_src0/34;
106
+ ax3 = (global char *) ((global char *) src0_q + offset_src0*sizeof(char)*QK8_0);
107
+ ad3 = (global half *) ((global char *) src0_d + offset_src0*sizeof(half));
108
+
109
+ const short ix = get_sub_group_local_id()/4;
110
+ const short il = get_sub_group_local_id()%4;
111
+
112
+ global float * yb = y + ix*QK8_0 + il*NB_Q8_0;
113
+
114
+ float8 yl;
115
+ float8 qv;
116
+ float4 sumf = 0.f;
117
+ float sumq = 0.f;
118
+ global char * qs;
119
+
120
+ // each thread handles NB_Q8_0 quants at a time
121
+ for (int ib = ix; ib < nb; ib += N_SIMDWIDTH/4) {
122
+ yl = vload8(0, yb);
123
+
124
+ qs = ax0 + ib*sizeof(char)*QK8_0 + il*NB_Q8_0;
125
+ qv = convert_float8(vload8(0, qs));
126
+ sumq = 0;
127
+ sumq += qv.s0*yl.s0;
128
+ sumq += qv.s1*yl.s1;
129
+ sumq += qv.s2*yl.s2;
130
+ sumq += qv.s3*yl.s3;
131
+ sumq += qv.s4*yl.s4;
132
+ sumq += qv.s5*yl.s5;
133
+ sumq += qv.s6*yl.s6;
134
+ sumq += qv.s7*yl.s7;
135
+ sumf.s0 += sumq*ad0[ib];
136
+
137
+ qs = ax1 + ib*sizeof(char)*QK8_0 + il*NB_Q8_0;
138
+ qv = convert_float8(vload8(0, qs));
139
+ sumq = 0;
140
+ sumq += qv.s0*yl.s0;
141
+ sumq += qv.s1*yl.s1;
142
+ sumq += qv.s2*yl.s2;
143
+ sumq += qv.s3*yl.s3;
144
+ sumq += qv.s4*yl.s4;
145
+ sumq += qv.s5*yl.s5;
146
+ sumq += qv.s6*yl.s6;
147
+ sumq += qv.s7*yl.s7;
148
+ sumf.s1 += sumq*ad1[ib];
149
+
150
+ qs = ax2 + ib*sizeof(char)*QK8_0 + il*NB_Q8_0;
151
+ qv = convert_float8(vload8(0, qs));
152
+ sumq = 0;
153
+ sumq += qv.s0*yl.s0;
154
+ sumq += qv.s1*yl.s1;
155
+ sumq += qv.s2*yl.s2;
156
+ sumq += qv.s3*yl.s3;
157
+ sumq += qv.s4*yl.s4;
158
+ sumq += qv.s5*yl.s5;
159
+ sumq += qv.s6*yl.s6;
160
+ sumq += qv.s7*yl.s7;
161
+ sumf.s2 += sumq*ad2[ib];
162
+
163
+ qs = ax3 + ib*sizeof(char)*QK8_0 + il*NB_Q8_0;
164
+ qv = convert_float8(vload8(0, qs));
165
+ sumq = 0;
166
+ sumq += qv.s0*yl.s0;
167
+ sumq += qv.s1*yl.s1;
168
+ sumq += qv.s2*yl.s2;
169
+ sumq += qv.s3*yl.s3;
170
+ sumq += qv.s4*yl.s4;
171
+ sumq += qv.s5*yl.s5;
172
+ sumq += qv.s6*yl.s6;
173
+ sumq += qv.s7*yl.s7;
174
+ sumf.s3 += sumq*ad3[ib];
175
+
176
+ yb += N_SIMDWIDTH*NB_Q8_0;
177
+ }
178
+
179
+ global float * dst_f32 = (global float *) dst + (ulong)im*ne0*ne1 + (ulong)r1*ne0;
180
+
181
+ float4 tot = (float4)(
182
+ sub_group_reduce_add(sumf.s0),
183
+ sub_group_reduce_add(sumf.s1),
184
+ sub_group_reduce_add(sumf.s2),
185
+ sub_group_reduce_add(sumf.s3)
186
+ );
187
+
188
+ if (get_sub_group_local_id() == 0) {
189
+ if (first_row + 0 < ne01) {
190
+ dst_f32[first_row + 0] = tot.s0;
191
+ }
192
+ if (first_row + 1 < ne01) {
193
+ dst_f32[first_row + 1] = tot.s1;
194
+ }
195
+ if (first_row + 2 < ne01) {
196
+ dst_f32[first_row + 2] = tot.s2;
197
+ }
198
+ if (first_row + 3 < ne01) {
199
+ dst_f32[first_row + 3] = tot.s3;
200
+ }
201
+ }
202
+ }
llama.cpp/ggml/src/ggml-opencl/kernels/neg.cl ADDED
@@ -0,0 +1,125 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #pragma OPENCL EXTENSION cl_khr_fp16 : enable
2
+
3
+ kernel void kernel_neg_f32(
4
+ global const float * src0,
5
+ ulong offset0,
6
+ global float * dst,
7
+ ulong offsetd,
8
+ int n
9
+ ) {
10
+ if (get_global_id(0) >= n) {
11
+ return;
12
+ }
13
+ src0 = (global float*)((global char*)src0 + offset0);
14
+ dst = (global float*)((global char*)dst + offsetd);
15
+
16
+ dst[get_global_id(0)] = -src0[get_global_id(0)];
17
+ }
18
+
19
+ kernel void kernel_neg_f32_4(
20
+ global const float4 * src0,
21
+ ulong offset0,
22
+ global float4 * dst,
23
+ ulong offsetd,
24
+ int n
25
+ ) {
26
+ if (get_global_id(0) >= n) {
27
+ return;
28
+ }
29
+ src0 = (global float4*)((global char*)src0 + offset0);
30
+ dst = (global float4*)((global char*)dst + offsetd);
31
+
32
+ dst[get_global_id(0)] = -src0[get_global_id(0)];
33
+ }
34
+
35
+ kernel void kernel_neg_f16(
36
+ global const half * src0,
37
+ ulong offset0,
38
+ global half * dst,
39
+ ulong offsetd,
40
+ int n
41
+ ) {
42
+ if (get_global_id(0) >= n) {
43
+ return;
44
+ }
45
+ src0 = (global half*)((global char*)src0 + offset0);
46
+ dst = (global half*)((global char*)dst + offsetd);
47
+
48
+ dst[get_global_id(0)] = -src0[get_global_id(0)];
49
+ }
50
+
51
+ kernel void kernel_neg_f16_4(
52
+ global const half4 * src0,
53
+ ulong offset0,
54
+ global half4 * dst,
55
+ ulong offsetd,
56
+ int n
57
+ ) {
58
+ if (get_global_id(0) >= n) {
59
+ return;
60
+ }
61
+ src0 = (global half4*)((global char*)src0 + offset0);
62
+ dst = (global half4*)((global char*)dst + offsetd);
63
+
64
+ dst[get_global_id(0)] = -src0[get_global_id(0)];
65
+ }
66
+
67
+ kernel void kernel_neg_f32_nc(
68
+ global const char * src0,
69
+ ulong offset0,
70
+ global char * dst,
71
+ ulong offsetd,
72
+ int ne00,
73
+ ulong nb00,
74
+ ulong nb01,
75
+ ulong nb02,
76
+ ulong nb03,
77
+ ulong nb0,
78
+ ulong nb1,
79
+ ulong nb2,
80
+ ulong nb3
81
+ ) {
82
+ src0 = src0 + offset0;
83
+ dst = dst + offsetd;
84
+
85
+ const int i3 = get_group_id(2);
86
+ const int i2 = get_group_id(1);
87
+ const int i1 = get_group_id(0);
88
+
89
+ for (int i0 = get_local_id(0); i0 < ne00; i0 += get_local_size(0)) {
90
+ global const float * x = (global const float *)(src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);
91
+ global float * y = (global float *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
92
+
93
+ *y = -*x;
94
+ }
95
+ }
96
+
97
+ kernel void kernel_neg_f16_nc(
98
+ global const char * src0,
99
+ ulong offset0,
100
+ global char * dst,
101
+ ulong offsetd,
102
+ int ne00,
103
+ ulong nb00,
104
+ ulong nb01,
105
+ ulong nb02,
106
+ ulong nb03,
107
+ ulong nb0,
108
+ ulong nb1,
109
+ ulong nb2,
110
+ ulong nb3
111
+ ) {
112
+ src0 = src0 + offset0;
113
+ dst = dst + offsetd;
114
+
115
+ const int i3 = get_group_id(2);
116
+ const int i2 = get_group_id(1);
117
+ const int i1 = get_group_id(0);
118
+
119
+ for (int i0 = get_local_id(0); i0 < ne00; i0 += get_local_size(0)) {
120
+ global const half * x = (global const half *)(src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);
121
+ global half * y = (global half *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
122
+
123
+ *y = -*x;
124
+ }
125
+ }
llama.cpp/ggml/src/ggml-opencl/kernels/norm.cl ADDED
@@ -0,0 +1,164 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #pragma OPENCL EXTENSION cl_khr_fp16 : enable
2
+
3
+ #ifdef cl_intel_required_subgroup_size
4
+ #pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
5
+ #define INTEL_GPU 1
6
+ #define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
7
+ #define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
8
+ #elif defined(cl_qcom_reqd_sub_group_size)
9
+ #pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
10
+ #define ADRENO_GPU 1
11
+ #define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
12
+ #define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
13
+ #endif
14
+
15
+ //------------------------------------------------------------------------------
16
+ // norm
17
+ //------------------------------------------------------------------------------
18
+ kernel void kernel_norm(
19
+ global void * src0,
20
+ ulong offset0,
21
+ global float * dst,
22
+ ulong offsetd,
23
+ int ne00,
24
+ int ne01,
25
+ int ne02,
26
+ int ne03,
27
+ ulong nb00,
28
+ ulong nb01,
29
+ ulong nb02,
30
+ ulong nb03,
31
+ float eps,
32
+ local float * sum
33
+ ) {
34
+ src0 = (global void*)((global char*)src0 + offset0);
35
+ dst = (global void*)((global char*)dst + offsetd);
36
+
37
+ int i03 = get_group_id(2);
38
+ int i02 = get_group_id(1);
39
+ int i01 = get_group_id(0);
40
+
41
+ global float * x = (global float *) ((global char *) src0 + i03*nb03 + i02*nb02 + i01*nb01);
42
+
43
+ // MEAN
44
+ // parallel sum
45
+ sum[get_local_id(0)] = 0.0f;
46
+ for (int i00 = get_local_id(0); i00 < ne00; i00 += get_local_size(0)) {
47
+ // this kernel handles float, nb00/4 translates byte offset to element offset
48
+ sum[get_local_id(0)] += x[i00*nb00/4];
49
+ }
50
+ // reduce
51
+ barrier(CLK_LOCAL_MEM_FENCE);
52
+ for (uint i = get_local_size(0)/2; i > 0; i /= 2) {
53
+ if (get_local_id(0) < i) {
54
+ sum[get_local_id(0)] += sum[get_local_id(0) + i];
55
+ }
56
+ barrier(CLK_LOCAL_MEM_FENCE);
57
+ }
58
+ float mean = sum[0] / ne00;
59
+
60
+ // recenter and VARIANCE
61
+ barrier(CLK_LOCAL_MEM_FENCE);
62
+ global float * y = dst + i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;
63
+ sum[get_local_id(0)] = 0.0f;
64
+ for (int i00 = get_local_id(0); i00 < ne00; i00 += get_local_size(0)) {
65
+ // this kernel handles float, nb00/4 translates byte offset to element offset
66
+ y[i00] = x[i00*nb00/4] - mean;
67
+ sum[get_local_id(0)] += y[i00] * y[i00];
68
+ }
69
+
70
+ // reduce
71
+ barrier(CLK_LOCAL_MEM_FENCE);
72
+ for (uint i = get_local_size(0)/2; i > 0; i /= 2) {
73
+ if (get_local_id(0) < i) {
74
+ sum[get_local_id(0)] += sum[get_local_id(0) + i];
75
+ }
76
+ barrier(CLK_LOCAL_MEM_FENCE);
77
+ }
78
+ float variance = sum[0] / ne00;
79
+
80
+ float scale = 1.0f/sqrt(variance + eps);
81
+ for (int i00 = get_local_id(0); i00 < ne00; i00 += get_local_size(0)) {
82
+ y[i00] = y[i00] * scale;
83
+ }
84
+ }
85
+
86
+ //------------------------------------------------------------------------------
87
+ // norm_mul_add
88
+ //------------------------------------------------------------------------------
89
+ #ifdef INTEL_GPU
90
+ REQD_SUBGROUP_SIZE_32
91
+ #elif defined (ADRENO_GPU)
92
+ REQD_SUBGROUP_SIZE_64
93
+ #endif
94
+ kernel void kernel_norm_mul_add(
95
+ global char * src0_ptr, ulong src0_offset,
96
+ global char * src1_ptr, ulong src1_offset,
97
+ global char * src2_ptr, ulong src2_offset,
98
+ global char * dst_ptr, ulong dst_offset,
99
+ int ne00, int ne01, int ne02, int ne03,
100
+ ulong nb01, ulong nb02, ulong nb03,
101
+ int ne10, int ne11, int ne12, int ne13,
102
+ ulong nb11, ulong nb12, ulong nb13,
103
+ int ne20, int ne21, int ne22, int ne23,
104
+ ulong nb21, ulong nb22, ulong nb23,
105
+ ulong nbd1, ulong nbd2, ulong nbd3,
106
+ float eps,
107
+ local float2 * sums
108
+ ) {
109
+ const int i03 = get_group_id(2);
110
+ const int i02 = get_group_id(1);
111
+ const int i01 = get_group_id(0);
112
+
113
+ global float4 * x = (global float4 *)(src0_ptr + src0_offset + i01*nb01 + i02*nb02 + i03*nb03);
114
+ global float4 * w = (global float4 *)(src1_ptr + src1_offset + (i01%ne11)*nb11 + (i02%ne12)*nb12 + (i03%ne13)*nb13);
115
+ global float4 * b = (global float4 *)(src2_ptr + src2_offset + (i01%ne21)*nb21 + (i02%ne22)*nb22 + (i03%ne23)*nb23);
116
+ global float4 * y = (global float4 *)(dst_ptr + dst_offset + i01*nbd1 + i02*nbd2 + i03*nbd3);
117
+
118
+ float p_sum = 0.0f;
119
+ float p_sum_sq = 0.0f;
120
+
121
+ const int n_chunks = ne00 / 4;
122
+ for (int i00 = get_local_id(0); i00 < n_chunks; i00 += get_local_size(0)) {
123
+ float4 val = x[i00];
124
+ p_sum += val.x + val.y + val.z + val.w;
125
+ p_sum_sq += dot(val, val);
126
+ }
127
+
128
+ p_sum = sub_group_reduce_add(p_sum);
129
+ p_sum_sq = sub_group_reduce_add(p_sum_sq);
130
+
131
+ if (get_sub_group_local_id() == 0) {
132
+ sums[get_sub_group_id()] = (float2)(p_sum, p_sum_sq);
133
+ }
134
+ barrier(CLK_LOCAL_MEM_FENCE);
135
+
136
+ if (get_local_id(0) == 0) {
137
+ float sum = 0.0f;
138
+ float sum_sq = 0.0f;
139
+ for (uint i = 0; i < get_num_sub_groups(); ++i) {
140
+ float2 s = sums[i];
141
+ sum += s.x;
142
+ sum_sq += s.y;
143
+ }
144
+
145
+ const float inv_ne00 = 1.0f / (float)ne00;
146
+ const float mean = sum * inv_ne00;
147
+ const float variance = mad(-mean, mean, sum_sq * inv_ne00);
148
+
149
+ sums[0] = (float2)(mean, rsqrt(variance + eps));
150
+ }
151
+ barrier(CLK_LOCAL_MEM_FENCE);
152
+
153
+ const float2 mean_scale = sums[0];
154
+ const float mean = mean_scale.x;
155
+ const float scale = mean_scale.y;
156
+ const float neg_mean_scale = -mean * scale;
157
+
158
+ for (int i00 = get_local_id(0); i00 < n_chunks; i00 += get_local_size(0)) {
159
+ const int w_idx = ne10 > 1 ? i00 : 0;
160
+ const int b_idx = ne20 > 1 ? i00 : 0;
161
+ const float4 norm_x = mad(x[i00], (float4)scale, (float4)neg_mean_scale);
162
+ y[i00] = mad(norm_x, w[w_idx], b[b_idx]);
163
+ }
164
+ }
llama.cpp/ggml/src/ggml-opencl/kernels/pad.cl ADDED
@@ -0,0 +1,39 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ kernel void kernel_pad(
2
+ global void * src0,
3
+ ulong offset0,
4
+ global void * dst,
5
+ ulong offsetd,
6
+ int ne00, int ne01, int ne02, int ne03,
7
+ ulong nb00, ulong nb01, ulong nb02, ulong nb03,
8
+ int ne0, int ne1, int ne2, int ne3,
9
+ ulong nb0, ulong nb1, ulong nb2, ulong nb3,
10
+ int lp0, int rp0,
11
+ int lp1, int rp1,
12
+ int lp2, int rp2,
13
+ int lp3, int rp3
14
+ ) {
15
+ src0 = (global float*)((global char*)src0 + offset0);
16
+ dst = (global float*)((global char*)dst + offsetd);
17
+
18
+ int i0 = get_global_id(0);
19
+ int i1 = get_group_id(1);
20
+ int i2 = get_group_id(2) % ne2;
21
+ int i3 = get_group_id(2) / ne2;
22
+
23
+ if (i0 >= ne0 || i1 >= ne1 || i2 >= ne2 || i3 >= ne3) {
24
+ return;
25
+ }
26
+
27
+ uint src0_idx = (i3 - lp3)*nb03 + (i2 - lp2)*nb02 + (i1 - lp1)*nb01 + (i0 - lp0)*nb00;
28
+ uint dst_idx = i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0;
29
+
30
+ global float * src0_ptr = (global float *)((global char *)src0 + src0_idx);
31
+ global float * dst_ptr = (global float *)((global char *)dst + dst_idx);
32
+
33
+ bool in_src_bounds = (i0 >= lp0 && i0 < ne0 - rp0) &&
34
+ (i1 >= lp1 && i1 < ne1 - rp1) &&
35
+ (i2 >= lp2 && i2 < ne2 - rp2) &&
36
+ (i3 >= lp3 && i3 < ne3 - rp3);
37
+
38
+ *dst_ptr = in_src_bounds ? *src0_ptr : 0.0f;
39
+ }
llama.cpp/ggml/src/ggml-opencl/kernels/relu.cl ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #pragma OPENCL EXTENSION cl_khr_fp16 : enable
2
+
3
+ //------------------------------------------------------------------------------
4
+ // relu
5
+ //------------------------------------------------------------------------------
6
+ kernel void kernel_relu(
7
+ global float * src0,
8
+ ulong offset0,
9
+ global float * dst,
10
+ ulong offsetd
11
+ ) {
12
+ src0 = (global float*)((global char*)src0 + offset0);
13
+ dst = (global float*)((global char*)dst + offsetd);
14
+
15
+ dst[get_global_id(0)] = fmax(0.0f, src0[get_global_id(0)]);
16
+ }
llama.cpp/ggml/src/ggml-opencl/kernels/repeat.cl ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ kernel void kernel_repeat_f32(
2
+ global const char * src0,
3
+ ulong offset0,
4
+ global char * dst,
5
+ ulong offsetd,
6
+ int ne00,
7
+ int ne01,
8
+ int ne02,
9
+ int ne03,
10
+ ulong nb00,
11
+ ulong nb01,
12
+ ulong nb02,
13
+ ulong nb03,
14
+ int ne0,
15
+ ulong nb0,
16
+ ulong nb1,
17
+ ulong nb2,
18
+ ulong nb3
19
+ ) {
20
+ src0 = src0 + offset0;
21
+ dst = dst + offsetd;
22
+
23
+ const int i3 = get_group_id(2);
24
+ const int i2 = get_group_id(1);
25
+ const int i1 = get_group_id(0);
26
+
27
+ const int i03 = i3%ne03;
28
+ const int i02 = i2%ne02;
29
+ const int i01 = i1%ne01;
30
+
31
+ global const char * src0_ptr = src0 + i03*nb03 + i02*nb02 + i01*nb01;
32
+ global char * dst_ptr = dst + i3*nb3 + i2*nb2 + i1*nb1;
33
+
34
+ for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) {
35
+ const int i00 = i0%ne00;
36
+ *((global float *)(dst_ptr + i0*nb0)) = *((global float *)(src0_ptr + i00*nb00));
37
+ }
38
+ }
llama.cpp/ggml/src/ggml-opencl/kernels/rms_norm.cl ADDED
@@ -0,0 +1,190 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #pragma OPENCL EXTENSION cl_khr_fp16 : enable
2
+
3
+ #ifdef cl_intel_subgroups
4
+ #pragma OPENCL EXTENSION cl_intel_subgroups : enable
5
+ #else
6
+ #pragma OPENCL EXTENSION cl_khr_subgroups : enable
7
+ #endif
8
+
9
+ #ifdef cl_intel_required_subgroup_size
10
+ #pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
11
+ #define INTEL_GPU 1
12
+ #define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
13
+ #define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
14
+ #elif defined(cl_qcom_reqd_sub_group_size)
15
+ #pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
16
+ #define ADRENO_GPU 1
17
+ #define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
18
+ #define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
19
+ #endif
20
+
21
+ //------------------------------------------------------------------------------
22
+ // rms_norm
23
+ //------------------------------------------------------------------------------
24
+ // This kernel depends on subgroup size.
25
+ #ifdef INTEL_GPU
26
+ REQD_SUBGROUP_SIZE_32
27
+ #elif defined (ADRENO_GPU)
28
+ REQD_SUBGROUP_SIZE_64
29
+ #endif
30
+ kernel void kernel_rms_norm(
31
+ global void * src0,
32
+ ulong offset0,
33
+ global float * dst,
34
+ ulong offsetd,
35
+ int ne00,
36
+ int ne01,
37
+ int ne02,
38
+ int ne03,
39
+ ulong nb01,
40
+ ulong nb02,
41
+ ulong nb03,
42
+ float eps,
43
+ local float * sum // Note, the size depends on number of subgroups
44
+ ) {
45
+ src0 = (global void*)((global char*)src0 + offset0);
46
+ dst = (global float*)((global char*)dst + offsetd);
47
+
48
+ int i03 = get_group_id(2);
49
+ int i02 = get_group_id(1);
50
+ int i01 = get_group_id(0);
51
+
52
+ global float4 * x = (global float4 *) ((global char *) src0 + i03*nb03 + i02*nb02 + i01*nb01);
53
+ global float * x_scalar = (global float *) x;
54
+ float4 sumf = 0;
55
+ float all_sum = 0;
56
+
57
+ // parallel sum
58
+ for (int i00 = get_local_id(0); i00 < ne00/4; i00 += get_local_size(0)) {
59
+ sumf += x[i00] * x[i00];
60
+ }
61
+ all_sum = sumf.s0 + sumf.s1 + sumf.s2 + sumf.s3;
62
+ all_sum = sub_group_reduce_add(all_sum);
63
+ if (get_sub_group_local_id() == 0) {
64
+ sum[get_sub_group_id()] = all_sum;
65
+ }
66
+
67
+ barrier(CLK_LOCAL_MEM_FENCE);
68
+ // broadcast
69
+ for (uint i = get_local_size(0) / get_max_sub_group_size() / 2; i > 0; i /= 2) {
70
+ if (get_local_id(0) < i) {
71
+ sum[get_local_id(0)] += sum[get_local_id(0) + i];
72
+ }
73
+ }
74
+ if (get_local_id(0) == 0) {
75
+ for (int i = 4 * (ne00 / 4); i < ne00; i++) {
76
+ sum[0] += x_scalar[i];
77
+ }
78
+ sum[0] /= ne00;
79
+ }
80
+
81
+ barrier(CLK_LOCAL_MEM_FENCE);
82
+
83
+ const float mean = sum[0];
84
+ const float scale = 1.0f/sqrt(mean + eps);
85
+
86
+ global float4 * y = (global float4 *) (dst + i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00);
87
+ global float * y_scalar = (global float *) y;
88
+ for (int i00 = get_local_id(0); i00 < ne00/4; i00 += get_local_size(0)) {
89
+ y[i00] = x[i00] * scale;
90
+ }
91
+ if (get_local_id(0) == 0) {
92
+ for (int i00 = 4 * (ne00 / 4); i00 < ne00; i00++) {
93
+ y_scalar[i00] = x_scalar[i00] * scale;
94
+ }
95
+ }
96
+ }
97
+
98
+ //------------------------------------------------------------------------------
99
+ // rms_norm_mul
100
+ //------------------------------------------------------------------------------
101
+ #ifdef INTEL_GPU
102
+ REQD_SUBGROUP_SIZE_32
103
+ #elif defined (ADRENO_GPU)
104
+ REQD_SUBGROUP_SIZE_64
105
+ #endif
106
+ kernel void kernel_rms_norm_mul(
107
+ global char * src0,
108
+ ulong offset0,
109
+ global char * src1,
110
+ ulong offset1,
111
+ global char * dst,
112
+ ulong offsetd,
113
+ int ne00,
114
+ int ne01,
115
+ int ne02,
116
+ int ne03,
117
+ ulong nb01,
118
+ ulong nb02,
119
+ ulong nb03,
120
+ int ne10,
121
+ int ne11,
122
+ int ne12,
123
+ int ne13,
124
+ ulong nb11,
125
+ ulong nb12,
126
+ ulong nb13,
127
+ ulong nb1,
128
+ ulong nb2,
129
+ ulong nb3,
130
+ float eps,
131
+ local float * sum
132
+ ) {
133
+ src0 = src0 + offset0;
134
+ src1 = src1 + offset1;
135
+ dst = dst + offsetd;
136
+
137
+ // The size of sum is sizeof(float)*subgroup_size.
138
+ // Each subgroup writes its partial sum to this array.
139
+ // So the number of subgroups per workgroup for this kernel cannot exceed the subgroup size.
140
+ // This is generally true -
141
+ // for subgroup size 64, workgroup size should be less than 4096 (the max is usually 1024).
142
+ if (get_sub_group_id() == 0) {
143
+ sum[get_sub_group_local_id()] = 0.0f;
144
+ }
145
+
146
+ int i03 = get_group_id(2);
147
+ int i02 = get_group_id(1);
148
+ int i01 = get_group_id(0);
149
+
150
+ global float4 * x = (global float4 *) (src0 + i03*nb03 + i02*nb02 + i01*nb01);
151
+ global float4 * f = (global float4 *) (src1 + (i03%ne13)*nb13 + (i02%ne12)*nb12 + (i01%ne11)*nb11);
152
+
153
+ float sumf = 0;
154
+
155
+ // parallel sum
156
+ for (int i00 = get_local_id(0); i00 < ne00/4; i00 += get_local_size(0)) {
157
+ sumf += dot(x[i00], x[i00]);
158
+ }
159
+ sumf = sub_group_reduce_add(sumf);
160
+
161
+ barrier(CLK_LOCAL_MEM_FENCE);
162
+
163
+ if (get_sub_group_local_id() == 0) {
164
+ sum[get_sub_group_id()] = sumf;
165
+ }
166
+
167
+ barrier(CLK_LOCAL_MEM_FENCE);
168
+
169
+ //for (uint i = get_local_size(0) / get_max_sub_group_size() / 2; i > 0; i /= 2) {
170
+ // if (get_local_id(0) < i) {
171
+ // sum[get_local_id(0)] += sum[get_local_id(0) + i];
172
+ // }
173
+ //}
174
+ //if (get_local_id(0) == 0) {
175
+ // sum[0] /= ne00;
176
+ //}
177
+
178
+ //barrier(CLK_LOCAL_MEM_FENCE);
179
+
180
+ sumf = sum[get_sub_group_local_id()];
181
+ sumf = sub_group_reduce_add(sumf);
182
+
183
+ float mean = sumf / ne00;
184
+ float scale = 1.0f/sqrt(mean + eps);
185
+
186
+ global float4 * y = (global float4 *) (dst + i03*nb3 + i02*nb2 + i01*nb1);
187
+ for (int i00 = get_local_id(0); i00 < ne00/4; i00 += get_local_size(0)) {
188
+ y[i00] = (x[i00] * scale) * f[i00%(ne10/4)];
189
+ }
190
+ }
llama.cpp/ggml/src/ggml-opencl/kernels/rope.cl ADDED
@@ -0,0 +1,747 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #pragma OPENCL EXTENSION cl_khr_fp16 : enable
2
+
3
+ //------------------------------------------------------------------------------
4
+ // kernel_rope
5
+ //------------------------------------------------------------------------------
6
+ float rope_yarn_ramp(float low, float high, int i0) {
7
+ const float y = (i0 / 2 - low) / max(0.001f, high - low);
8
+ return 1.0f - min(1.0f, max(0.0f, y));
9
+ }
10
+
11
+ // YaRN algorithm based on LlamaYaRNScaledRotaryEmbedding.py from https://github.com/jquesnelle/yarn
12
+ // MIT licensed. Copyright (c) 2023 Jeffrey Quesnelle and Bowen Peng.
13
+ float2 rope_yarn(
14
+ float theta_extrap, float freq_scale, float2 corr_dims, int i0, float ext_factor, float mscale
15
+ ) {
16
+ // Get n-d rotational scaling corrected for extrapolation
17
+ float theta_interp = freq_scale * theta_extrap;
18
+ float theta = theta_interp;
19
+ if (ext_factor != 0.0f) {
20
+ float ramp_mix = rope_yarn_ramp(corr_dims.s0, corr_dims.s1, i0) * ext_factor;
21
+ theta = theta_interp * (1 - ramp_mix) + theta_extrap * ramp_mix;
22
+
23
+ // Get n-d magnitude scaling corrected for interpolation
24
+ mscale *= 1.0f + 0.1f * log(1.0f / freq_scale);
25
+ }
26
+ return (float2)(cos(theta) * mscale, sin(theta) * mscale);
27
+ }
28
+
29
+ // Apparently solving `n_rot = 2pi * x * base^((2 * max_pos_emb) / n_dims)` for x, we get
30
+ // `corr_fac(n_rot) = n_dims * log(max_pos_emb / (n_rot * 2pi)) / (2 * log(base))`
31
+ float rope_yarn_corr_factor(int n_dims, int n_ctx_orig, float n_rot, float base) {
32
+ return n_dims * log(n_ctx_orig / (n_rot * 2 * M_PI_F)) / (2 * log(base));
33
+ }
34
+
35
+ float2 rope_yarn_corr_dims(
36
+ int n_dims, int n_ctx_orig, float freq_base, float beta_fast, float beta_slow
37
+ ) {
38
+ // start and end correction dims
39
+ return (float2)(
40
+ max(0.0f, floor(rope_yarn_corr_factor(n_dims, n_ctx_orig, beta_fast, freq_base))),
41
+ min(n_dims - 1.0f, ceil(rope_yarn_corr_factor(n_dims, n_ctx_orig, beta_slow, freq_base)))
42
+ );
43
+ }
44
+
45
+ kernel void kernel_rope_norm_f32(
46
+ global void * src0,
47
+ ulong offset0,
48
+ global int * src1,
49
+ ulong offset1,
50
+ global float * src2,
51
+ ulong offset2,
52
+ global float * dst,
53
+ ulong offsetd,
54
+ int ne00,
55
+ int ne01,
56
+ int ne02,
57
+ int ne03,
58
+ ulong nb00,
59
+ ulong nb01,
60
+ ulong nb02,
61
+ ulong nb03,
62
+ int ne0,
63
+ int ne1,
64
+ int ne2,
65
+ int ne3,
66
+ ulong nb0,
67
+ ulong nb1,
68
+ ulong nb2,
69
+ ulong nb3,
70
+ int n_past,
71
+ int n_dims,
72
+ int n_ctx_orig,
73
+ float freq_base,
74
+ float freq_scale,
75
+ float ext_factor,
76
+ float attn_factor,
77
+ float beta_fast,
78
+ float beta_slow
79
+ ) {
80
+ src0 = (global void*)((global char*)src0 + offset0);
81
+ src1 = (global int*)((global char*)src1 + offset1);
82
+ src2 = (global float*)((global char*)src2 + offset2);
83
+ dst = (global float*)((global char*)dst + offsetd);
84
+
85
+ int i3 = get_group_id(2);
86
+ int i2 = get_group_id(1);
87
+ int i1 = get_group_id(0);
88
+
89
+ float2 corr_dims = rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow);
90
+
91
+ global int * pos = src1;
92
+
93
+ float theta_base = (float) pos[i2];
94
+ float inv_ndims = -1.f/n_dims;
95
+
96
+ for (int i0 = 2*get_local_id(0); i0 < ne0; i0 += 2*get_local_size(0)) {
97
+ if (i0 < n_dims) {
98
+ int ic = i0/2;
99
+
100
+ float theta = theta_base * pow(freq_base, inv_ndims*i0);
101
+
102
+ float freq_factor = src2 != src0 ? src2[ic] : 1.0f;
103
+
104
+ float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor);
105
+
106
+ global float * src = (global float *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);
107
+ global float * dst_data = (global float *)((global char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
108
+
109
+ float x0 = src[0];
110
+ float x1 = src[1];
111
+
112
+ dst_data[0] = x0*cos_sin_theta.s0 - x1*cos_sin_theta.s1;
113
+ dst_data[1] = x0*cos_sin_theta.s1 + x1*cos_sin_theta.s0;
114
+ } else {
115
+ global float * src = (global float *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);
116
+ global float * dst_data = (global float *)((global char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
117
+
118
+ dst_data[0] = src[0];
119
+ dst_data[1] = src[1];
120
+ }
121
+ }
122
+ }
123
+
124
+ kernel void kernel_rope_norm_f16(
125
+ global void * src0,
126
+ ulong offset0,
127
+ global int * src1,
128
+ ulong offset1,
129
+ global float * src2,
130
+ ulong offset2,
131
+ global float * dst,
132
+ ulong offsetd,
133
+ int ne00,
134
+ int ne01,
135
+ int ne02,
136
+ int ne03,
137
+ ulong nb00,
138
+ ulong nb01,
139
+ ulong nb02,
140
+ ulong nb03,
141
+ int ne0,
142
+ int ne1,
143
+ int ne2,
144
+ int ne3,
145
+ ulong nb0,
146
+ ulong nb1,
147
+ ulong nb2,
148
+ ulong nb3,
149
+ int n_past,
150
+ int n_dims,
151
+ int n_ctx_orig,
152
+ float freq_base,
153
+ float freq_scale,
154
+ float ext_factor,
155
+ float attn_factor,
156
+ float beta_fast,
157
+ float beta_slow
158
+ ) {
159
+ src0 = (global void*)((global char*)src0 + offset0);
160
+ src1 = (global int*)((global char*)src1 + offset1);
161
+ src2 = (global float*)((global char*)src2 + offset2);
162
+ dst = (global float*)((global char*)dst + offsetd);
163
+
164
+ int i3 = get_group_id(2);
165
+ int i2 = get_group_id(1);
166
+ int i1 = get_group_id(0);
167
+
168
+ float2 corr_dims = rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow);
169
+
170
+ global int * pos = src1;
171
+
172
+ float theta_base = (float) pos[i2];
173
+ float inv_ndims = -1.f/n_dims;
174
+
175
+ for (int i0 = 2*get_local_id(0); i0 < ne0; i0 += 2*get_local_size(0)) {
176
+ if (i0 < n_dims) {
177
+ int ic = i0/2;
178
+
179
+ float theta = theta_base * pow(freq_base, inv_ndims*i0);
180
+
181
+ float freq_factor = src2 != src0 ? src2[ic] : 1.0f;
182
+
183
+ float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor);
184
+
185
+ global half * src = (global half *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);
186
+ global half * dst_data = (global half *)((global char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
187
+
188
+ float x0 = src[0];
189
+ float x1 = src[1];
190
+
191
+ dst_data[0] = x0*cos_sin_theta.s0 - x1*cos_sin_theta.s1;
192
+ dst_data[1] = x0*cos_sin_theta.s1 + x1*cos_sin_theta.s0;
193
+ } else {
194
+ global half * src = (global half *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);
195
+ global half * dst_data = (global half *)((global char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
196
+
197
+ dst_data[0] = src[0];
198
+ dst_data[1] = src[1];
199
+ }
200
+ }
201
+ }
202
+
203
+ kernel void kernel_rope_neox_f32(
204
+ global void * src0,
205
+ ulong offset0,
206
+ global int * src1,
207
+ ulong offset1,
208
+ global float * src2,
209
+ ulong offset2,
210
+ global float * dst,
211
+ ulong offsetd,
212
+ int ne00,
213
+ int ne01,
214
+ int ne02,
215
+ int ne03,
216
+ ulong nb00,
217
+ ulong nb01,
218
+ ulong nb02,
219
+ ulong nb03,
220
+ int ne0,
221
+ int ne1,
222
+ int ne2,
223
+ int ne3,
224
+ ulong nb0,
225
+ ulong nb1,
226
+ ulong nb2,
227
+ ulong nb3,
228
+ int n_past,
229
+ int n_dims,
230
+ int n_ctx_orig,
231
+ float freq_base,
232
+ float freq_scale,
233
+ float ext_factor,
234
+ float attn_factor,
235
+ float beta_fast,
236
+ float beta_slow
237
+ ) {
238
+ src0 = (global void*)((global char*)src0 + offset0);
239
+ src1 = (global int*)((global char*)src1 + offset1);
240
+ src2 = (global float*)((global char*)src2 + offset2);
241
+ dst = (global float*)((global char*)dst + offsetd);
242
+
243
+ int i3 = get_group_id(2);
244
+ int i2 = get_group_id(1);
245
+ int i1 = get_group_id(0);
246
+
247
+ float2 corr_dims = rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow);
248
+
249
+ global int * pos = src1;
250
+
251
+ float theta_base = (float) pos[i2];
252
+ float inv_ndims = -1.f/n_dims;
253
+
254
+ for (int i0 = 2*get_local_id(0); i0 < ne0; i0 += 2*get_local_size(0)) {
255
+ if (i0 < n_dims) {
256
+ int ic = i0/2;
257
+
258
+ const float theta = theta_base * pow(freq_base, inv_ndims*i0);
259
+
260
+ const float freq_factor = src2 != src0 ? src2[ic] : 1.0f;
261
+
262
+ float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor);
263
+
264
+ global float * src = (global float *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + ic*nb00);
265
+ global float * dst_data = (global float *)((global char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + ic*nb0);
266
+
267
+ const float x0 = src[0];
268
+ const float x1 = src[n_dims/2];
269
+
270
+ dst_data[0] = x0*cos_sin_theta.s0 - x1*cos_sin_theta.s1;
271
+ dst_data[n_dims/2] = x0*cos_sin_theta.s1 + x1*cos_sin_theta.s0;
272
+ } else {
273
+ global float * const src = (global float *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);
274
+ global float * dst_data = (global float *)((global char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
275
+
276
+ dst_data[0] = src[0];
277
+ dst_data[1] = src[1];
278
+ }
279
+ }
280
+ }
281
+
282
+ kernel void kernel_rope_neox_f16(
283
+ global void * src0,
284
+ ulong offset0,
285
+ global int * src1,
286
+ ulong offset1,
287
+ global float * src2,
288
+ ulong offset2,
289
+ global float * dst,
290
+ ulong offsetd,
291
+ int ne00,
292
+ int ne01,
293
+ int ne02,
294
+ int ne03,
295
+ ulong nb00,
296
+ ulong nb01,
297
+ ulong nb02,
298
+ ulong nb03,
299
+ int ne0,
300
+ int ne1,
301
+ int ne2,
302
+ int ne3,
303
+ ulong nb0,
304
+ ulong nb1,
305
+ ulong nb2,
306
+ ulong nb3,
307
+ int n_past,
308
+ int n_dims,
309
+ int n_ctx_orig,
310
+ float freq_base,
311
+ float freq_scale,
312
+ float ext_factor,
313
+ float attn_factor,
314
+ float beta_fast,
315
+ float beta_slow
316
+ ) {
317
+ src0 = (global void*)((global char*)src0 + offset0);
318
+ src1 = (global int*)((global char*)src1 + offset1);
319
+ src2 = (global float*)((global char*)src2 + offset2);
320
+ dst = (global float*)((global char*)dst + offsetd);
321
+
322
+ int i3 = get_group_id(2);
323
+ int i2 = get_group_id(1);
324
+ int i1 = get_group_id(0);
325
+
326
+ float2 corr_dims = rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow);
327
+
328
+ global int * pos = src1;
329
+
330
+ float theta_base = (float) pos[i2];
331
+ float inv_ndims = -1.f/n_dims;
332
+
333
+ for (int i0 = 2*get_local_id(0); i0 < ne0; i0 += 2*get_local_size(0)) {
334
+ if (i0 < n_dims) {
335
+ int ic = i0/2;
336
+
337
+ const float theta = theta_base * pow(freq_base, inv_ndims*i0);
338
+
339
+ const float freq_factor = src2 != src0 ? src2[ic] : 1.0f;
340
+
341
+ float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor);
342
+
343
+ global half * src = (global half *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + ic*nb00);
344
+ global half * dst_data = (global half *)((global char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + ic*nb0);
345
+
346
+ const float x0 = src[0];
347
+ const float x1 = src[n_dims/2];
348
+
349
+ dst_data[0] = x0*cos_sin_theta.s0 - x1*cos_sin_theta.s1;
350
+ dst_data[n_dims/2] = x0*cos_sin_theta.s1 + x1*cos_sin_theta.s0;
351
+ } else {
352
+ global half * const src = (global half *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);
353
+ global half * dst_data = (global half *)((global char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
354
+
355
+ dst_data[0] = src[0];
356
+ dst_data[1] = src[1];
357
+ }
358
+ }
359
+ }
360
+
361
+ kernel void kernel_rope_multi_f32(
362
+ global void * src0,
363
+ ulong offset0,
364
+ global int * src1,
365
+ ulong offset1,
366
+ global float * src2,
367
+ ulong offset2,
368
+ global float * dst,
369
+ ulong offsetd,
370
+ int ne00,
371
+ int ne01,
372
+ int ne02,
373
+ int ne03,
374
+ ulong nb00,
375
+ ulong nb01,
376
+ ulong nb02,
377
+ ulong nb03,
378
+ int ne0,
379
+ int ne1,
380
+ int ne2,
381
+ int ne3,
382
+ ulong nb0,
383
+ ulong nb1,
384
+ ulong nb2,
385
+ ulong nb3,
386
+ int n_past,
387
+ int n_dims,
388
+ int n_ctx_orig,
389
+ float freq_base,
390
+ float freq_scale,
391
+ float ext_factor,
392
+ float attn_factor,
393
+ float beta_fast,
394
+ float beta_slow,
395
+ int4 sections,
396
+ int is_imrope
397
+ ) {
398
+ src0 = (global void*)((global char*)src0 + offset0);
399
+ src1 = (global int*)((global char*)src1 + offset1);
400
+ src2 = (global float*)((global char*)src2 + offset2);
401
+ dst = (global float*)((global char*)dst + offsetd);
402
+
403
+ int i3 = get_group_id(2);
404
+ int i2 = get_group_id(1);
405
+ int i1 = get_group_id(0);
406
+
407
+ float2 corr_dims = rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow);
408
+
409
+ global int * pos = src1;
410
+
411
+ const int sect_dims = sections.s0 + sections.s1 + sections.s2 + sections.s3;
412
+ const int sec_w = sections.s1 + sections.s0;
413
+
414
+ float inv_ndims = -1.f/n_dims;
415
+
416
+ for (int i0 = 2*get_local_id(0); i0 < ne0; i0 += 2*get_local_size(0)) {
417
+ if (i0 < n_dims) {
418
+ int ic = i0/2;
419
+
420
+ const int sector = (i0 / 2) % sect_dims;
421
+ float theta_base = 0.0f;
422
+
423
+ if (is_imrope) {
424
+ if (sector % 3 == 1 && sector < 3 * sections.s1) { // h
425
+ theta_base = (float) pos[i2 + ne02 * 1];
426
+ } else if (sector % 3 == 2 && sector < 3 * sections.s2) { // w
427
+ theta_base = (float) pos[i2 + ne02 * 2];
428
+ } else if (sector % 3 == 0 && sector < 3 * sections.s0) { // t
429
+ theta_base = (float) pos[i2 + ne02 * 0];
430
+ } else { // e
431
+ theta_base = (float) pos[i2 + ne02 * 3];
432
+ }
433
+ } else {
434
+ if (sector < sections.s0) {
435
+ theta_base = pos[i2];
436
+ }
437
+ else if (sector >= sections.s0 && sector < sec_w) {
438
+ theta_base = pos[i2 + ne2 * 1];
439
+ }
440
+ else if (sector >= sec_w && sector < sec_w + sections.s2) {
441
+ theta_base = pos[i2 + ne2 * 2];
442
+ }
443
+ else if (sector >= sec_w + sections.s2) {
444
+ theta_base = pos[i2 + ne2 * 3];
445
+ }
446
+ }
447
+
448
+ const float theta = theta_base * pow(freq_base, inv_ndims*i0);
449
+
450
+ const float freq_factor = src2 != src0 ? src2[ic] : 1.0f;
451
+
452
+ float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor);
453
+
454
+ global float * src = (global float *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + ic*nb00);
455
+ global float * dst_data = (global float *)((global char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + ic*nb0);
456
+
457
+ const float x0 = src[0];
458
+ const float x1 = src[n_dims/2];
459
+
460
+ dst_data[0] = x0*cos_sin_theta.s0 - x1*cos_sin_theta.s1;
461
+ dst_data[n_dims/2] = x0*cos_sin_theta.s1 + x1*cos_sin_theta.s0;
462
+ } else {
463
+ global float * const src = (global float *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);
464
+ global float * dst_data = (global float *)((global char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
465
+
466
+ dst_data[0] = src[0];
467
+ dst_data[1] = src[1];
468
+ }
469
+ }
470
+ }
471
+
472
+ kernel void kernel_rope_multi_f16(
473
+ global void * src0,
474
+ ulong offset0,
475
+ global int * src1,
476
+ ulong offset1,
477
+ global float * src2,
478
+ ulong offset2,
479
+ global half * dst,
480
+ ulong offsetd,
481
+ int ne00,
482
+ int ne01,
483
+ int ne02,
484
+ int ne03,
485
+ ulong nb00,
486
+ ulong nb01,
487
+ ulong nb02,
488
+ ulong nb03,
489
+ int ne0,
490
+ int ne1,
491
+ int ne2,
492
+ int ne3,
493
+ ulong nb0,
494
+ ulong nb1,
495
+ ulong nb2,
496
+ ulong nb3,
497
+ int n_past,
498
+ int n_dims,
499
+ int n_ctx_orig,
500
+ float freq_base,
501
+ float freq_scale,
502
+ float ext_factor,
503
+ float attn_factor,
504
+ float beta_fast,
505
+ float beta_slow,
506
+ int4 sections,
507
+ int is_imrope
508
+ ) {
509
+ src0 = (global void*)((global char*)src0 + offset0);
510
+ src1 = (global int*)((global char*)src1 + offset1);
511
+ src2 = (global float*)((global char*)src2 + offset2);
512
+ dst = (global float*)((global char*)dst + offsetd);
513
+
514
+ int i3 = get_group_id(2);
515
+ int i2 = get_group_id(1);
516
+ int i1 = get_group_id(0);
517
+
518
+ float2 corr_dims = rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow);
519
+
520
+ global int * pos = src1;
521
+
522
+ const int sect_dims = sections.s0 + sections.s1 + sections.s2 + sections.s3;
523
+ const int sec_w = sections.s1 + sections.s0;
524
+
525
+ float inv_ndims = -1.f/n_dims;
526
+
527
+ for (int i0 = 2*get_local_id(0); i0 < ne0; i0 += 2*get_local_size(0)) {
528
+ if (i0 < n_dims) {
529
+ int ic = i0/2;
530
+
531
+ const int sector = (i0 / 2) % sect_dims;
532
+ float theta_base = 0.0f;
533
+
534
+ if (is_imrope) {
535
+ if (sector % 3 == 1 && sector < 3 * sections.s1) { // h
536
+ theta_base = (float) pos[i2 + ne02 * 1];
537
+ } else if (sector % 3 == 2 && sector < 3 * sections.s2) { // w
538
+ theta_base = (float) pos[i2 + ne02 * 2];
539
+ } else if (sector % 3 == 0 && sector < 3 * sections.s0) { // t
540
+ theta_base = (float) pos[i2 + ne02 * 0];
541
+ } else { // e
542
+ theta_base = (float) pos[i2 + ne02 * 3];
543
+ }
544
+ } else {
545
+ if (sector < sections.s0) {
546
+ theta_base = pos[i2];
547
+ }
548
+ else if (sector >= sections.s0 && sector < sec_w) {
549
+ theta_base = pos[i2 + ne2 * 1];
550
+ }
551
+ else if (sector >= sec_w && sector < sec_w + sections.s2) {
552
+ theta_base = pos[i2 + ne2 * 2];
553
+ }
554
+ else if (sector >= sec_w + sections.s2) {
555
+ theta_base = pos[i2 + ne2 * 3];
556
+ }
557
+ }
558
+
559
+ const float theta = theta_base * pow(freq_base, inv_ndims*i0);
560
+
561
+ const float freq_factor = src2 != src0 ? src2[ic] : 1.0f;
562
+
563
+ float2 cos_sin_theta = rope_yarn(theta/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor);
564
+
565
+ global half * src = (global half *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + ic*nb00);
566
+ global half * dst_data = (global half *)((global char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + ic*nb0);
567
+
568
+ const float x0 = src[0];
569
+ const float x1 = src[n_dims/2];
570
+
571
+ dst_data[0] = x0*cos_sin_theta.s0 - x1*cos_sin_theta.s1;
572
+ dst_data[n_dims/2] = x0*cos_sin_theta.s1 + x1*cos_sin_theta.s0;
573
+ } else {
574
+ global half * const src = (global half *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);
575
+ global half * dst_data = (global half *)((global char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
576
+
577
+ dst_data[0] = src[0];
578
+ dst_data[1] = src[1];
579
+ }
580
+ }
581
+ }
582
+
583
+ kernel void kernel_rope_vision_f32(
584
+ global void * src0,
585
+ ulong offset0,
586
+ global int * src1,
587
+ ulong offset1,
588
+ global float * src2,
589
+ ulong offset2,
590
+ global float * dst,
591
+ ulong offsetd,
592
+ int ne00,
593
+ int ne01,
594
+ int ne02,
595
+ int ne03,
596
+ ulong nb00,
597
+ ulong nb01,
598
+ ulong nb02,
599
+ ulong nb03,
600
+ int ne0,
601
+ int ne1,
602
+ int ne2,
603
+ int ne3,
604
+ ulong nb0,
605
+ ulong nb1,
606
+ ulong nb2,
607
+ ulong nb3,
608
+ int n_past,
609
+ int n_dims,
610
+ int n_ctx_orig,
611
+ float freq_base,
612
+ float freq_scale,
613
+ float ext_factor,
614
+ float attn_factor,
615
+ float beta_fast,
616
+ float beta_slow,
617
+ int4 sections
618
+ ) {
619
+ src0 = (global void*)((global char*)src0 + offset0);
620
+ src1 = (global int*)((global char*)src1 + offset1);
621
+ src2 = (global float*)((global char*)src2 + offset2);
622
+ dst = (global float*)((global char*)dst + offsetd);
623
+
624
+ int i3 = get_group_id(2);
625
+ int i2 = get_group_id(1);
626
+ int i1 = get_group_id(0);
627
+
628
+ float2 corr_dims = rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow);
629
+
630
+ global int * pos = src1;
631
+
632
+ const int sect_dims = sections.s0 + sections.s1;
633
+ const int sec_w = sections.s1 + sections.s0;
634
+
635
+ float inv_ndims = -1.f/n_dims;
636
+
637
+ for (int i0 = 2*get_local_id(0); i0 < ne0; i0 += 2*get_local_size(0)) {
638
+ int ic = i0/2;
639
+
640
+ const int sector = (i0/2) % sect_dims;
641
+ float theta_base = 0.0f;
642
+
643
+ if (sector < sections.s0) {
644
+ const int p = sector;
645
+ theta_base = pos[i2] * pow(freq_base, inv_ndims*2.0f*p);
646
+ } else if (sector >= sections.s0 && sector < sec_w) {
647
+ const int p = sector - sections.s0;
648
+ theta_base = pos[i2 + ne2] * pow(freq_base, inv_ndims*2.0f*p);
649
+ }
650
+
651
+ const float freq_factor = src2 != src0 ? src2[ic] : 1.0f;
652
+
653
+ float2 cos_sin_theta = rope_yarn(theta_base/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor);
654
+
655
+ global float * src = (global float *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + ic*nb00);
656
+ global float * dst_data = (global float *)((global char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + ic*nb0);
657
+
658
+ const float x0 = src[0];
659
+ const float x1 = src[n_dims];
660
+
661
+ dst_data[0] = x0*cos_sin_theta.s0 - x1*cos_sin_theta.s1;
662
+ dst_data[n_dims] = x0*cos_sin_theta.s1 + x1*cos_sin_theta.s0;
663
+ }
664
+ }
665
+
666
+ kernel void kernel_rope_vision_f16(
667
+ global void * src0,
668
+ ulong offset0,
669
+ global int * src1,
670
+ ulong offset1,
671
+ global float * src2,
672
+ ulong offset2,
673
+ global half * dst,
674
+ ulong offsetd,
675
+ int ne00,
676
+ int ne01,
677
+ int ne02,
678
+ int ne03,
679
+ ulong nb00,
680
+ ulong nb01,
681
+ ulong nb02,
682
+ ulong nb03,
683
+ int ne0,
684
+ int ne1,
685
+ int ne2,
686
+ int ne3,
687
+ ulong nb0,
688
+ ulong nb1,
689
+ ulong nb2,
690
+ ulong nb3,
691
+ int n_past,
692
+ int n_dims,
693
+ int n_ctx_orig,
694
+ float freq_base,
695
+ float freq_scale,
696
+ float ext_factor,
697
+ float attn_factor,
698
+ float beta_fast,
699
+ float beta_slow,
700
+ int4 sections
701
+ ) {
702
+ src0 = (global void*)((global char*)src0 + offset0);
703
+ src1 = (global int*)((global char*)src1 + offset1);
704
+ src2 = (global float*)((global char*)src2 + offset2);
705
+ dst = (global float*)((global char*)dst + offsetd);
706
+
707
+ int i3 = get_group_id(2);
708
+ int i2 = get_group_id(1);
709
+ int i1 = get_group_id(0);
710
+
711
+ float2 corr_dims = rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow);
712
+
713
+ global int * pos = src1;
714
+
715
+ const int sect_dims = sections.s0 + sections.s1;
716
+ const int sec_w = sections.s1 + sections.s0;
717
+
718
+ float inv_ndims = -1.f/n_dims;
719
+
720
+ for (int i0 = 2*get_local_id(0); i0 < ne0; i0 += 2*get_local_size(0)) {
721
+ int ic = i0/2;
722
+
723
+ const int sector = (i0/2) % sect_dims;
724
+ float theta_base = 0.0f;
725
+
726
+ if (sector < sections.s0) {
727
+ const int p = sector;
728
+ theta_base = pos[i2] * pow(freq_base, inv_ndims*2.0f*p);
729
+ } else if (sector >= sections.s0 && sector < sec_w) {
730
+ const int p = sector - sections.s0;
731
+ theta_base = pos[i2 + ne2] * pow(freq_base, inv_ndims*2.0f*p);
732
+ }
733
+
734
+ const float freq_factor = src2 != src0 ? src2[ic] : 1.0f;
735
+
736
+ float2 cos_sin_theta = rope_yarn(theta_base/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor);
737
+
738
+ global half * src = (global half *)((global char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + ic*nb00);
739
+ global half * dst_data = (global half *)((global char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + ic*nb0);
740
+
741
+ const float x0 = src[0];
742
+ const float x1 = src[n_dims];
743
+
744
+ dst_data[0] = x0*cos_sin_theta.s0 - x1*cos_sin_theta.s1;
745
+ dst_data[n_dims] = x0*cos_sin_theta.s1 + x1*cos_sin_theta.s0;
746
+ }
747
+ }
llama.cpp/ggml/src/ggml-opencl/kernels/scale.cl ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #pragma OPENCL EXTENSION cl_khr_fp16 : enable
2
+
3
+ kernel void kernel_scale_f32(
4
+ global float * src0,
5
+ ulong offset0,
6
+ global float * dst,
7
+ ulong offsetd,
8
+ float scale,
9
+ float bias
10
+ ) {
11
+ src0 = (global float*)((global char*)src0 + offset0);
12
+ dst = (global float*)((global char*)dst + offsetd);
13
+ dst[get_global_id(0)] = src0[get_global_id(0)] * scale + bias;
14
+ }
15
+
16
+ kernel void kernel_scale_f32_4(
17
+ global float4 * src0,
18
+ ulong offset0,
19
+ global float4 * dst,
20
+ ulong offsetd,
21
+ float scale,
22
+ float bias
23
+ ) {
24
+ src0 = (global float4*)((global char*)src0 + offset0);
25
+ dst = (global float4*)((global char*)dst + offsetd);
26
+ dst[get_global_id(0)] = src0[get_global_id(0)] * scale + bias;
27
+ }
llama.cpp/ggml/src/ggml-opencl/kernels/set_rows.cl ADDED
@@ -0,0 +1,708 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #pragma OPENCL EXTENSION cl_khr_fp16 : enable
2
+
3
+ // v = { mp, L, d }
4
+ inline uint fastdiv(uint n, uint4 v) {
5
+ uint msbs;
6
+ msbs = mul_hi(n, v.s0);
7
+ return (msbs + n) >> v.s1;
8
+ }
9
+ inline uint fastmod(uint n, uint4 v) {
10
+ uint q = fastdiv(n, v);
11
+ return n - q * v.s2;
12
+ }
13
+
14
+ kernel void kernel_set_rows_f32_i64(
15
+ global char * src0,
16
+ ulong offset0,
17
+ global char * src1,
18
+ ulong offset1,
19
+ global char * dst,
20
+ ulong offsetd,
21
+ int ne01,
22
+ ulong nb01,
23
+ ulong nb02,
24
+ ulong nb03,
25
+ uint4 ne11,
26
+ uint4 ne12,
27
+ ulong nb10,
28
+ ulong nb11,
29
+ ulong nb12,
30
+ int nblk0,
31
+ ulong nb1,
32
+ ulong nb2,
33
+ ulong nb3
34
+ ) {
35
+ src0 = src0 + offset0;
36
+ src1 = src1 + offset1;
37
+ dst = dst + offsetd;
38
+
39
+ int i03 = get_group_id(2);
40
+ int i02 = get_group_id(1);
41
+ int i01 = get_group_id(0)*get_local_size(1) + get_local_id(1);
42
+
43
+ if (i01 >= ne01) {
44
+ return;
45
+ }
46
+
47
+ //int i12 = i03%ne12;
48
+ //int i11 = i02%ne11;
49
+ int i12 = fastmod(i03, ne12);
50
+ int i11 = fastmod(i02, ne11);
51
+
52
+ int i10 = i01;
53
+ long i1 = ((global long *)(src1 + i10*nb10 + i11*nb11 + i12*nb12))[0];
54
+
55
+ global float * dst_row = (global float *) (dst + i1*nb1 + i02*nb2 + i03*nb3);
56
+ global float * src_row = (global float *) (src0 + i01*nb01 + i02*nb02 + i03*nb03);
57
+
58
+ for (int ind = get_local_id(0); ind < nblk0; ind += get_local_size(0)) {
59
+ dst_row[ind] = (float)src_row[ind];
60
+ }
61
+ }
62
+
63
+ kernel void kernel_set_rows_f16_i64(
64
+ global char * src0,
65
+ ulong offset0,
66
+ global char * src1,
67
+ ulong offset1,
68
+ global char * dst,
69
+ ulong offsetd,
70
+ int ne01,
71
+ ulong nb01,
72
+ ulong nb02,
73
+ ulong nb03,
74
+ uint4 ne11,
75
+ uint4 ne12,
76
+ ulong nb10,
77
+ ulong nb11,
78
+ ulong nb12,
79
+ int nblk0,
80
+ ulong nb1,
81
+ ulong nb2,
82
+ ulong nb3
83
+ ) {
84
+ src0 = src0 + offset0;
85
+ src1 = src1 + offset1;
86
+ dst = dst + offsetd;
87
+
88
+ int i03 = get_group_id(2);
89
+ int i02 = get_group_id(1);
90
+ int i01 = get_group_id(0)*get_local_size(1) + get_local_id(1);
91
+
92
+ if (i01 >= ne01) {
93
+ return;
94
+ }
95
+
96
+ //int i12 = i03%ne12;
97
+ //int i11 = i02%ne11;
98
+ int i12 = fastmod(i03, ne12);
99
+ int i11 = fastmod(i02, ne11);
100
+
101
+ int i10 = i01;
102
+ long i1 = ((global long *)(src1 + i10*nb10 + i11*nb11 + i12*nb12))[0];
103
+
104
+ global half * dst_row = (global half *) (dst + i1*nb1 + i02*nb2 + i03*nb3);
105
+ global float * src_row = (global float *) (src0 + i01*nb01 + i02*nb02 + i03*nb03);
106
+
107
+ for (int ind = get_local_id(0); ind < nblk0; ind += get_local_size(0)) {
108
+ dst_row[ind] = src_row[ind];
109
+ }
110
+ }
111
+
112
+ kernel void kernel_set_rows_f32_i32(
113
+ global char * src0,
114
+ ulong offset0,
115
+ global char * src1,
116
+ ulong offset1,
117
+ global char * dst,
118
+ ulong offsetd,
119
+ int ne01,
120
+ ulong nb01,
121
+ ulong nb02,
122
+ ulong nb03,
123
+ uint4 ne11,
124
+ uint4 ne12,
125
+ ulong nb10,
126
+ ulong nb11,
127
+ ulong nb12,
128
+ int nblk0,
129
+ ulong nb1,
130
+ ulong nb2,
131
+ ulong nb3
132
+ ) {
133
+ src0 = src0 + offset0;
134
+ src1 = src1 + offset1;
135
+ dst = dst + offsetd;
136
+
137
+ int i03 = get_group_id(2);
138
+ int i02 = get_group_id(1);
139
+ int i01 = get_group_id(0)*get_local_size(1) + get_local_id(1);
140
+
141
+ if (i01 >= ne01) {
142
+ return;
143
+ }
144
+
145
+ //int i12 = i03%ne12;
146
+ //int i11 = i02%ne11;
147
+ int i12 = fastmod(i03, ne12);
148
+ int i11 = fastmod(i02, ne11);
149
+
150
+ int i10 = i01;
151
+ int i1 = ((global int *)(src1 + i10*nb10 + i11*nb11 + i12*nb12))[0];
152
+
153
+ global float * dst_row = (global float *) (dst + i1*nb1 + i02*nb2 + i03*nb3);
154
+ global float * src_row = (global float *) (src0 + i01*nb01 + i02*nb02 + i03*nb03);
155
+
156
+ for (int ind = get_local_id(0); ind < nblk0; ind += get_local_size(0)) {
157
+ dst_row[ind] = (float)src_row[ind];
158
+ }
159
+ }
160
+
161
+ // f32 -> q8_0 quantize set_rows. Block = half d + char qs[32].
162
+ #define QK8_0 32
163
+
164
+ inline void quantize_q8_0_block(global float * x, global char * qs, global half * d_out) {
165
+ float amax = 0.0f;
166
+ for (int j = 0; j < QK8_0; j++) {
167
+ amax = fmax(amax, fabs(x[j]));
168
+ }
169
+
170
+ float d = amax / 127.0f;
171
+ float id = (d != 0.0f) ? 127.0f / amax : 0.0f;
172
+
173
+ vstore_half(d, 0, d_out);
174
+
175
+ for (int j = 0; j < QK8_0; j++) {
176
+ qs[j] = (char)((int)round(x[j] * id));
177
+ }
178
+ }
179
+
180
+ kernel void kernel_set_rows_q8_0_i64(
181
+ global char * src0,
182
+ ulong offset0,
183
+ global char * src1,
184
+ ulong offset1,
185
+ global char * dst,
186
+ ulong offsetd,
187
+ int ne01,
188
+ ulong nb01,
189
+ ulong nb02,
190
+ ulong nb03,
191
+ uint4 ne11,
192
+ uint4 ne12,
193
+ ulong nb10,
194
+ ulong nb11,
195
+ ulong nb12,
196
+ int nblk0,
197
+ ulong nb1,
198
+ ulong nb2,
199
+ ulong nb3
200
+ ) {
201
+ src0 = src0 + offset0;
202
+ src1 = src1 + offset1;
203
+ dst = dst + offsetd;
204
+
205
+ int i03 = get_group_id(2);
206
+ int i02 = get_group_id(1);
207
+ int i01 = get_group_id(0)*get_local_size(1) + get_local_id(1);
208
+
209
+ if (i01 >= ne01) {
210
+ return;
211
+ }
212
+
213
+ int i12 = fastmod(i03, ne12);
214
+ int i11 = fastmod(i02, ne11);
215
+
216
+ int i10 = i01;
217
+ long i1 = ((global long *)(src1 + i10*nb10 + i11*nb11 + i12*nb12))[0];
218
+
219
+ global char * dst_row = (global char *) (dst + i1*nb1 + i02*nb2 + i03*nb3);
220
+ global float * src_row = (global float *) (src0 + i01*nb01 + i02*nb02 + i03*nb03);
221
+
222
+ for (int blk = get_local_id(0); blk < nblk0; blk += get_local_size(0)) {
223
+ global float * x = src_row + blk * QK8_0;
224
+ global char * y = dst_row + blk * (2 + QK8_0);
225
+
226
+ quantize_q8_0_block(x, y + 2, (global half *)y);
227
+ }
228
+ }
229
+
230
+ kernel void kernel_set_rows_q8_0_i32(
231
+ global char * src0,
232
+ ulong offset0,
233
+ global char * src1,
234
+ ulong offset1,
235
+ global char * dst,
236
+ ulong offsetd,
237
+ int ne01,
238
+ ulong nb01,
239
+ ulong nb02,
240
+ ulong nb03,
241
+ uint4 ne11,
242
+ uint4 ne12,
243
+ ulong nb10,
244
+ ulong nb11,
245
+ ulong nb12,
246
+ int nblk0,
247
+ ulong nb1,
248
+ ulong nb2,
249
+ ulong nb3
250
+ ) {
251
+ src0 = src0 + offset0;
252
+ src1 = src1 + offset1;
253
+ dst = dst + offsetd;
254
+
255
+ int i03 = get_group_id(2);
256
+ int i02 = get_group_id(1);
257
+ int i01 = get_group_id(0)*get_local_size(1) + get_local_id(1);
258
+
259
+ if (i01 >= ne01) {
260
+ return;
261
+ }
262
+
263
+ int i12 = fastmod(i03, ne12);
264
+ int i11 = fastmod(i02, ne11);
265
+
266
+ int i10 = i01;
267
+ int i1 = ((global int *)(src1 + i10*nb10 + i11*nb11 + i12*nb12))[0];
268
+
269
+ global char * dst_row = (global char *) (dst + i1*nb1 + i02*nb2 + i03*nb3);
270
+ global float * src_row = (global float *) (src0 + i01*nb01 + i02*nb02 + i03*nb03);
271
+
272
+ for (int blk = get_local_id(0); blk < nblk0; blk += get_local_size(0)) {
273
+ global float * x = src_row + blk * QK8_0;
274
+ global char * y = dst_row + blk * (2 + QK8_0);
275
+
276
+ quantize_q8_0_block(x, y + 2, (global half *)y);
277
+ }
278
+ }
279
+
280
+ // SoA q8_0 variants. dst_q: int8[QK8_0] per block; dst_d: fp16 scale per block.
281
+ // Layout matches kernel_convert_block_q8_0; block index follows dst element order.
282
+ kernel void kernel_set_rows_q8_0_soa_i64(
283
+ global char * src0,
284
+ ulong offset0,
285
+ global char * src1,
286
+ ulong offset1,
287
+ global char * dst_q,
288
+ ulong offset_q,
289
+ global char * dst_d,
290
+ ulong offset_d,
291
+ int ne01,
292
+ ulong nb01,
293
+ ulong nb02,
294
+ ulong nb03,
295
+ uint4 ne11,
296
+ uint4 ne12,
297
+ ulong nb10,
298
+ ulong nb11,
299
+ ulong nb12,
300
+ int nblk0,
301
+ int ne1_dst,
302
+ int ne2_dst,
303
+ int ne3_dst
304
+ ) {
305
+ src0 = src0 + offset0;
306
+ src1 = src1 + offset1;
307
+ dst_q = dst_q + offset_q;
308
+ dst_d = dst_d + offset_d;
309
+
310
+ int i03 = get_group_id(2);
311
+ int i02 = get_group_id(1);
312
+ int i01 = get_group_id(0)*get_local_size(1) + get_local_id(1);
313
+
314
+ if (i01 >= ne01) {
315
+ return;
316
+ }
317
+
318
+ int i12 = fastmod(i03, ne12);
319
+ int i11 = fastmod(i02, ne11);
320
+
321
+ int i10 = i01;
322
+ long i1 = ((global long *)(src1 + i10*nb10 + i11*nb11 + i12*nb12))[0];
323
+
324
+ long row_blk_base = ((long)i03 * ne2_dst * ne1_dst + (long)i02 * ne1_dst + i1) * nblk0;
325
+
326
+ global half * d_row = (global half *)(dst_d) + row_blk_base;
327
+ global char * q_row = (global char *)(dst_q) + row_blk_base * QK8_0;
328
+ global float * src_row = (global float *)(src0 + i01*nb01 + i02*nb02 + i03*nb03);
329
+
330
+ for (int blk = get_local_id(0); blk < nblk0; blk += get_local_size(0)) {
331
+ global float * x = src_row + blk * QK8_0;
332
+ global char * q = q_row + blk * QK8_0;
333
+
334
+ quantize_q8_0_block(x, q, d_row + blk);
335
+ }
336
+ }
337
+
338
+ kernel void kernel_set_rows_q8_0_soa_i32(
339
+ global char * src0,
340
+ ulong offset0,
341
+ global char * src1,
342
+ ulong offset1,
343
+ global char * dst_q,
344
+ ulong offset_q,
345
+ global char * dst_d,
346
+ ulong offset_d,
347
+ int ne01,
348
+ ulong nb01,
349
+ ulong nb02,
350
+ ulong nb03,
351
+ uint4 ne11,
352
+ uint4 ne12,
353
+ ulong nb10,
354
+ ulong nb11,
355
+ ulong nb12,
356
+ int nblk0,
357
+ int ne1_dst,
358
+ int ne2_dst,
359
+ int ne3_dst
360
+ ) {
361
+ src0 = src0 + offset0;
362
+ src1 = src1 + offset1;
363
+ dst_q = dst_q + offset_q;
364
+ dst_d = dst_d + offset_d;
365
+
366
+ int i03 = get_group_id(2);
367
+ int i02 = get_group_id(1);
368
+ int i01 = get_group_id(0)*get_local_size(1) + get_local_id(1);
369
+
370
+ if (i01 >= ne01) {
371
+ return;
372
+ }
373
+
374
+ int i12 = fastmod(i03, ne12);
375
+ int i11 = fastmod(i02, ne11);
376
+
377
+ int i10 = i01;
378
+ int i1 = ((global int *)(src1 + i10*nb10 + i11*nb11 + i12*nb12))[0];
379
+
380
+ long row_blk_base = ((long)i03 * ne2_dst * ne1_dst + (long)i02 * ne1_dst + i1) * nblk0;
381
+
382
+ global half * d_row = (global half *)(dst_d) + row_blk_base;
383
+ global char * q_row = (global char *)(dst_q) + row_blk_base * QK8_0;
384
+ global float * src_row = (global float *)(src0 + i01*nb01 + i02*nb02 + i03*nb03);
385
+
386
+ for (int blk = get_local_id(0); blk < nblk0; blk += get_local_size(0)) {
387
+ global float * x = src_row + blk * QK8_0;
388
+ global char * q = q_row + blk * QK8_0;
389
+
390
+ quantize_q8_0_block(x, q, d_row + blk);
391
+ }
392
+ }
393
+
394
+ kernel void kernel_set_rows_f16_i32(
395
+ global char * src0,
396
+ ulong offset0,
397
+ global char * src1,
398
+ ulong offset1,
399
+ global char * dst,
400
+ ulong offsetd,
401
+ int ne01,
402
+ ulong nb01,
403
+ ulong nb02,
404
+ ulong nb03,
405
+ uint4 ne11,
406
+ uint4 ne12,
407
+ ulong nb10,
408
+ ulong nb11,
409
+ ulong nb12,
410
+ int nblk0,
411
+ ulong nb1,
412
+ ulong nb2,
413
+ ulong nb3
414
+ ) {
415
+ src0 = src0 + offset0;
416
+ src1 = src1 + offset1;
417
+ dst = dst + offsetd;
418
+
419
+ int i03 = get_group_id(2);
420
+ int i02 = get_group_id(1);
421
+ int i01 = get_group_id(0)*get_local_size(1) + get_local_id(1);
422
+
423
+ if (i01 >= ne01) {
424
+ return;
425
+ }
426
+
427
+ //int i12 = i03%ne12;
428
+ //int i11 = i02%ne11;
429
+ int i12 = fastmod(i03, ne12);
430
+ int i11 = fastmod(i02, ne11);
431
+
432
+ int i10 = i01;
433
+ int i1 = ((global int *)(src1 + i10*nb10 + i11*nb11 + i12*nb12))[0];
434
+
435
+ global half * dst_row = (global half *) (dst + i1*nb1 + i02*nb2 + i03*nb3);
436
+ global float * src_row = (global float *) (src0 + i01*nb01 + i02*nb02 + i03*nb03);
437
+
438
+ for (int ind = get_local_id(0); ind < nblk0; ind += get_local_size(0)) {
439
+ dst_row[ind] = src_row[ind];
440
+ }
441
+ }
442
+
443
+ // f32 -> q4_0 quantize set_rows. Block = half d + uchar qs[16] (shuffled
444
+ // nibbles: qs[j] low/high = elem j / j+16).
445
+ // Dequant: val[i] = d * (nibble_i - 8)
446
+ // nblk0 = number of q4_0 blocks per row = ne00 / 32.
447
+ #define QK4_0 32
448
+ #define Q4_0_BLOCK_SIZE 18
449
+
450
+ inline void quantize_q4_0_block(global float * x, global uchar * qs, global half * d_out) {
451
+ // Find the signed value with the largest absolute magnitude (matches ggml ref).
452
+ float max = 0.0f;
453
+ float amax = 0.0f;
454
+ for (int j = 0; j < QK4_0; j++) {
455
+ float v = x[j];
456
+ float a = fabs(v);
457
+ if (a > amax) {
458
+ amax = a;
459
+ max = v;
460
+ }
461
+ }
462
+
463
+ float d = max / -8.0f;
464
+ float id = (d != 0.0f) ? 1.0f / d : 0.0f;
465
+
466
+ vstore_half(d, 0, d_out);
467
+
468
+ for (int j = 0; j < QK4_0/2; j++) {
469
+ float x0 = x[j] * id;
470
+ float x1 = x[j + QK4_0/2] * id;
471
+
472
+ int i0 = (int)(x0 + 8.5f);
473
+ int i1 = (int)(x1 + 8.5f);
474
+ if (i0 < 0) i0 = 0;
475
+ if (i0 > 15) i0 = 15;
476
+ if (i1 < 0) i1 = 0;
477
+ if (i1 > 15) i1 = 15;
478
+
479
+ qs[j] = (uchar)i0 | ((uchar)i1 << 4);
480
+ }
481
+ }
482
+
483
+ kernel void kernel_set_rows_q4_0_i64(
484
+ global char * src0,
485
+ ulong offset0,
486
+ global char * src1,
487
+ ulong offset1,
488
+ global char * dst,
489
+ ulong offsetd,
490
+ int ne01,
491
+ ulong nb01,
492
+ ulong nb02,
493
+ ulong nb03,
494
+ uint4 ne11,
495
+ uint4 ne12,
496
+ ulong nb10,
497
+ ulong nb11,
498
+ ulong nb12,
499
+ int nblk0,
500
+ ulong nb1,
501
+ ulong nb2,
502
+ ulong nb3
503
+ ) {
504
+ src0 = src0 + offset0;
505
+ src1 = src1 + offset1;
506
+ dst = dst + offsetd;
507
+
508
+ int i03 = get_group_id(2);
509
+ int i02 = get_group_id(1);
510
+ int i01 = get_group_id(0)*get_local_size(1) + get_local_id(1);
511
+
512
+ if (i01 >= ne01) {
513
+ return;
514
+ }
515
+
516
+ int i12 = fastmod(i03, ne12);
517
+ int i11 = fastmod(i02, ne11);
518
+
519
+ int i10 = i01;
520
+ long i1 = ((global long *)(src1 + i10*nb10 + i11*nb11 + i12*nb12))[0];
521
+
522
+ global char * dst_row = (global char *) (dst + i1*nb1 + i02*nb2 + i03*nb3);
523
+ global float * src_row = (global float *) (src0 + i01*nb01 + i02*nb02 + i03*nb03);
524
+
525
+ for (int blk = get_local_id(0); blk < nblk0; blk += get_local_size(0)) {
526
+ global float * x = src_row + blk * QK4_0;
527
+ global char * y = dst_row + blk * Q4_0_BLOCK_SIZE;
528
+ global half * yd = (global half *)(y);
529
+ global uchar * yqs = (global uchar *)(y + 2);
530
+
531
+ quantize_q4_0_block(x, yqs, yd);
532
+ }
533
+ }
534
+
535
+ kernel void kernel_set_rows_q4_0_i32(
536
+ global char * src0,
537
+ ulong offset0,
538
+ global char * src1,
539
+ ulong offset1,
540
+ global char * dst,
541
+ ulong offsetd,
542
+ int ne01,
543
+ ulong nb01,
544
+ ulong nb02,
545
+ ulong nb03,
546
+ uint4 ne11,
547
+ uint4 ne12,
548
+ ulong nb10,
549
+ ulong nb11,
550
+ ulong nb12,
551
+ int nblk0,
552
+ ulong nb1,
553
+ ulong nb2,
554
+ ulong nb3
555
+ ) {
556
+ src0 = src0 + offset0;
557
+ src1 = src1 + offset1;
558
+ dst = dst + offsetd;
559
+
560
+ int i03 = get_group_id(2);
561
+ int i02 = get_group_id(1);
562
+ int i01 = get_group_id(0)*get_local_size(1) + get_local_id(1);
563
+
564
+ if (i01 >= ne01) {
565
+ return;
566
+ }
567
+
568
+ int i12 = fastmod(i03, ne12);
569
+ int i11 = fastmod(i02, ne11);
570
+
571
+ int i10 = i01;
572
+ int i1 = ((global int *)(src1 + i10*nb10 + i11*nb11 + i12*nb12))[0];
573
+
574
+ global char * dst_row = (global char *) (dst + i1*nb1 + i02*nb2 + i03*nb3);
575
+ global float * src_row = (global float *) (src0 + i01*nb01 + i02*nb02 + i03*nb03);
576
+
577
+ for (int blk = get_local_id(0); blk < nblk0; blk += get_local_size(0)) {
578
+ global float * x = src_row + blk * QK4_0;
579
+ global char * y = dst_row + blk * Q4_0_BLOCK_SIZE;
580
+ global half * yd = (global half *)(y);
581
+ global uchar * yqs = (global uchar *)(y + 2);
582
+
583
+ quantize_q4_0_block(x, yqs, yd);
584
+ }
585
+ }
586
+
587
+ // SoA variants for q4_0 dst. Used when the backend has split block_q4_0 records
588
+ // into separate quant (dst_q) and scale (dst_d) sub-buffers — same pattern as
589
+ // the q8_0 SoA variants above.
590
+ //
591
+ // Layout (matches kernel_convert_block_q4_0, the "shuffled" variant):
592
+ // dst_q: contiguous 16 packed nibbles per block, block i at offset i * 16 bytes.
593
+ // dst_d: contiguous fp16 scales, block i at offset i * 2 bytes.
594
+ // Nibble layout inside each byte is unchanged from AoS: qs[j] low nibble = element j,
595
+ // qs[j] high nibble = element j+16. kernel_restore_block_q4_0 copies bytes as-is.
596
+ kernel void kernel_set_rows_q4_0_soa_i64(
597
+ global char * src0,
598
+ ulong offset0,
599
+ global char * src1,
600
+ ulong offset1,
601
+ global char * dst_q,
602
+ ulong offset_q,
603
+ global char * dst_d,
604
+ ulong offset_d,
605
+ int ne01,
606
+ ulong nb01,
607
+ ulong nb02,
608
+ ulong nb03,
609
+ uint4 ne11,
610
+ uint4 ne12,
611
+ ulong nb10,
612
+ ulong nb11,
613
+ ulong nb12,
614
+ int nblk0,
615
+ int ne1_dst,
616
+ int ne2_dst,
617
+ int ne3_dst
618
+ ) {
619
+ src0 = src0 + offset0;
620
+ src1 = src1 + offset1;
621
+ dst_q = dst_q + offset_q;
622
+ dst_d = dst_d + offset_d;
623
+
624
+ int i03 = get_group_id(2);
625
+ int i02 = get_group_id(1);
626
+ int i01 = get_group_id(0)*get_local_size(1) + get_local_id(1);
627
+
628
+ if (i01 >= ne01) {
629
+ return;
630
+ }
631
+
632
+ int i12 = fastmod(i03, ne12);
633
+ int i11 = fastmod(i02, ne11);
634
+
635
+ int i10 = i01;
636
+ long i1 = ((global long *)(src1 + i10*nb10 + i11*nb11 + i12*nb12))[0];
637
+
638
+ long row_blk_base = ((long)i03 * ne2_dst * ne1_dst + (long)i02 * ne1_dst + i1) * nblk0;
639
+
640
+ global half * d_row = (global half *)(dst_d) + row_blk_base;
641
+ global uchar * q_row = (global uchar *)(dst_q) + row_blk_base * (QK4_0/2);
642
+ global float * src_row = (global float *)(src0 + i01*nb01 + i02*nb02 + i03*nb03);
643
+
644
+ for (int blk = get_local_id(0); blk < nblk0; blk += get_local_size(0)) {
645
+ global float * x = src_row + blk * QK4_0;
646
+ global uchar * qs = q_row + blk * (QK4_0/2);
647
+ global half * d_bk = d_row + blk;
648
+
649
+ quantize_q4_0_block(x, qs, d_bk);
650
+ }
651
+ }
652
+
653
+ kernel void kernel_set_rows_q4_0_soa_i32(
654
+ global char * src0,
655
+ ulong offset0,
656
+ global char * src1,
657
+ ulong offset1,
658
+ global char * dst_q,
659
+ ulong offset_q,
660
+ global char * dst_d,
661
+ ulong offset_d,
662
+ int ne01,
663
+ ulong nb01,
664
+ ulong nb02,
665
+ ulong nb03,
666
+ uint4 ne11,
667
+ uint4 ne12,
668
+ ulong nb10,
669
+ ulong nb11,
670
+ ulong nb12,
671
+ int nblk0,
672
+ int ne1_dst,
673
+ int ne2_dst,
674
+ int ne3_dst
675
+ ) {
676
+ src0 = src0 + offset0;
677
+ src1 = src1 + offset1;
678
+ dst_q = dst_q + offset_q;
679
+ dst_d = dst_d + offset_d;
680
+
681
+ int i03 = get_group_id(2);
682
+ int i02 = get_group_id(1);
683
+ int i01 = get_group_id(0)*get_local_size(1) + get_local_id(1);
684
+
685
+ if (i01 >= ne01) {
686
+ return;
687
+ }
688
+
689
+ int i12 = fastmod(i03, ne12);
690
+ int i11 = fastmod(i02, ne11);
691
+
692
+ int i10 = i01;
693
+ int i1 = ((global int *)(src1 + i10*nb10 + i11*nb11 + i12*nb12))[0];
694
+
695
+ long row_blk_base = ((long)i03 * ne2_dst * ne1_dst + (long)i02 * ne1_dst + i1) * nblk0;
696
+
697
+ global half * d_row = (global half *)(dst_d) + row_blk_base;
698
+ global uchar * q_row = (global uchar *)(dst_q) + row_blk_base * (QK4_0/2);
699
+ global float * src_row = (global float *)(src0 + i01*nb01 + i02*nb02 + i03*nb03);
700
+
701
+ for (int blk = get_local_id(0); blk < nblk0; blk += get_local_size(0)) {
702
+ global float * x = src_row + blk * QK4_0;
703
+ global uchar * qs = q_row + blk * (QK4_0/2);
704
+ global half * d_bk = d_row + blk;
705
+
706
+ quantize_q4_0_block(x, qs, d_bk);
707
+ }
708
+ }
llama.cpp/ggml/src/ggml-opencl/kernels/sigmoid.cl ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #pragma OPENCL EXTENSION cl_khr_fp16 : enable
2
+
3
+ //------------------------------------------------------------------------------
4
+ // sigmoid
5
+ //------------------------------------------------------------------------------
6
+
7
+ kernel void kernel_sigmoid_f32(
8
+ global float * src0,
9
+ ulong offset0,
10
+ global float * dst,
11
+ ulong offsetd
12
+ ) {
13
+ src0 = (global float*)((global char*)src0 + offset0);
14
+ dst = (global float*)((global char*)dst + offsetd);
15
+
16
+ dst[get_global_id(0)] = 1.0f / (1.0f + exp(-src0[get_global_id(0)]));
17
+ }
18
+
19
+ kernel void kernel_sigmoid_f16(
20
+ global half * src0,
21
+ ulong offset0,
22
+ global half * dst,
23
+ ulong offsetd
24
+ ) {
25
+ src0 = (global half*)((global char*)src0 + offset0);
26
+ dst = (global half*)((global char*)dst + offsetd);
27
+
28
+ dst[get_global_id(0)] = 1.0f / (1.0f + exp(-src0[get_global_id(0)]));
29
+ }
llama.cpp/ggml/src/ggml-opencl/kernels/silu.cl ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #pragma OPENCL EXTENSION cl_khr_fp16 : enable
2
+
3
+ //------------------------------------------------------------------------------
4
+ // silu
5
+ //------------------------------------------------------------------------------
6
+ kernel void kernel_silu(
7
+ global float * src0,
8
+ ulong offset0,
9
+ global float * dst,
10
+ ulong offsetd
11
+ ) {
12
+ src0 = (global float*)((global char*)src0 + offset0);
13
+ dst = (global float*)((global char*)dst + offsetd);
14
+
15
+ float x = src0[get_global_id(0)];
16
+ dst[get_global_id(0)] = x / (1.0f + exp(-x));
17
+ }
18
+
19
+ kernel void kernel_silu_4(
20
+ global float4 * src0,
21
+ ulong offset0,
22
+ global float4 * dst,
23
+ ulong offsetd
24
+ ) {
25
+ src0 = (global float4*)((global char*)src0 + offset0);
26
+ dst = (global float4*)((global char*)dst + offsetd);
27
+
28
+ float4 x = src0[get_global_id(0)];
29
+ dst[get_global_id(0)] = x / (1.0f + exp(-x));
30
+ }
llama.cpp/ggml/src/ggml-opencl/kernels/softmax_4_f16.cl ADDED
@@ -0,0 +1,108 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #pragma OPENCL EXTENSION cl_khr_fp16 : enable
2
+
3
+ #ifdef cl_intel_subgroups
4
+ #pragma OPENCL EXTENSION cl_intel_subgroups : enable
5
+ #else
6
+ #pragma OPENCL EXTENSION cl_khr_subgroups : enable
7
+ #endif
8
+
9
+ #ifdef cl_intel_required_subgroup_size
10
+ #pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
11
+ #define INTEL_GPU 1
12
+ #define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
13
+ #define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
14
+ #elif defined(cl_qcom_reqd_sub_group_size)
15
+ #pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
16
+ #define ADRENO_GPU 1
17
+ #define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
18
+ #define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
19
+ #endif
20
+
21
+ #ifdef ADRENO_GPU
22
+ REQD_SUBGROUP_SIZE_64
23
+ #endif
24
+ kernel void kernel_soft_max_4_f16(
25
+ global char * src0,
26
+ ulong offset0,
27
+ global char * src1,
28
+ ulong offset1,
29
+ global char * src2,
30
+ ulong offset2,
31
+ global char * dst,
32
+ ulong offsetd,
33
+ int ne00,
34
+ ulong nb01,
35
+ ulong nb02,
36
+ ulong nb03,
37
+ int ne12,
38
+ int ne13,
39
+ ulong nb11,
40
+ ulong nb12,
41
+ ulong nb13,
42
+ ulong nb1,
43
+ ulong nb2,
44
+ ulong nb3,
45
+ float scale,
46
+ float max_bias,
47
+ float m0,
48
+ float m1,
49
+ int n_head_log2
50
+ ) {
51
+ src0 = src0 + offset0;
52
+ src1 = src1 + offset1;
53
+ src2 = src2 + offset2;
54
+ dst = dst + offsetd;
55
+
56
+ int i03 = get_group_id(2);
57
+ int i02 = get_group_id(1);
58
+ int i01 = get_group_id(0);
59
+
60
+ int i13 = i03%ne13;
61
+ int i12 = i02%ne12;
62
+ int i11 = i01;
63
+
64
+ global float4 * psrc4 = (global float4 *)(src0 + i01*nb01 + i02*nb02 + i03*nb03);
65
+ global half4 * pmask = src1 != src0 ? (global half4 *)(src1 + i11*nb11 + i12*nb12 + i13*nb13) : 0;
66
+ global float * psrc2 = src2 != src0 ? (global float *)(src2) : 0;
67
+ global float4 * pdst4 = (global float4 *)(dst + i01*nb1 + i02*nb2 + i03*nb3);
68
+
69
+ float slope = 1.0f;
70
+
71
+ // ALiBi
72
+ if (max_bias > 0.0f) {
73
+ int h = i02;
74
+
75
+ float base = h < n_head_log2 ? m0 : m1;
76
+ int exp = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1;
77
+
78
+ slope = pow(base, exp);
79
+ }
80
+
81
+ // parallel max
82
+ float4 lmax4 = psrc2 ? psrc2[i02] : -INFINITY;
83
+ for (int i00 = get_local_id(0); i00 < ne00/4; i00 += get_local_size(0)) {
84
+ lmax4 = fmax(lmax4, psrc4[i00]*scale + slope*(pmask ? convert_float4(pmask[i00]) : 0.0f));
85
+ }
86
+ float lmax = fmax(fmax(lmax4.s0, lmax4.s1), fmax(lmax4.s2, lmax4.s3));
87
+
88
+ const float max = sub_group_reduce_max(lmax);
89
+
90
+ // parallel sum
91
+ float4 lsum4 = 0.0f;
92
+ for (int i00 = get_local_id(0); i00 < ne00/4; i00 += get_local_size(0)) {
93
+ const float4 exp_psrc4 = exp((psrc4[i00]*scale + slope*(pmask ? convert_float4(pmask[i00]) : 0.0f)) - max);
94
+ lsum4 += exp_psrc4;
95
+ pdst4[i00] = exp_psrc4;
96
+ }
97
+ float lsum = lsum4.s0 + lsum4.s1 + lsum4.s2 + lsum4.s3;
98
+
99
+ float sum = sub_group_reduce_add(lsum);
100
+
101
+ if (psrc2) {
102
+ sum += exp(psrc2[i02] - max);
103
+ }
104
+
105
+ for (int i00 = get_local_id(0); i00 < ne00/4; i00 += get_local_size(0)) {
106
+ pdst4[i00] /= sum;
107
+ }
108
+ }
llama.cpp/ggml/src/ggml-opencl/kernels/softmax_4_f32.cl ADDED
@@ -0,0 +1,108 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #pragma OPENCL EXTENSION cl_khr_fp16 : enable
2
+
3
+ #ifdef cl_intel_subgroups
4
+ #pragma OPENCL EXTENSION cl_intel_subgroups : enable
5
+ #else
6
+ #pragma OPENCL EXTENSION cl_khr_subgroups : enable
7
+ #endif
8
+
9
+ #ifdef cl_intel_required_subgroup_size
10
+ #pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
11
+ #define INTEL_GPU 1
12
+ #define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
13
+ #define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
14
+ #elif defined(cl_qcom_reqd_sub_group_size)
15
+ #pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
16
+ #define ADRENO_GPU 1
17
+ #define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
18
+ #define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
19
+ #endif
20
+
21
+ #ifdef ADRENO_GPU
22
+ REQD_SUBGROUP_SIZE_64
23
+ #endif
24
+ kernel void kernel_soft_max_4(
25
+ global char * src0,
26
+ ulong offset0,
27
+ global char * src1,
28
+ ulong offset1,
29
+ global char * src2,
30
+ ulong offset2,
31
+ global char * dst,
32
+ ulong offsetd,
33
+ int ne00,
34
+ ulong nb01,
35
+ ulong nb02,
36
+ ulong nb03,
37
+ int ne12,
38
+ int ne13,
39
+ ulong nb11,
40
+ ulong nb12,
41
+ ulong nb13,
42
+ ulong nb1,
43
+ ulong nb2,
44
+ ulong nb3,
45
+ float scale,
46
+ float max_bias,
47
+ float m0,
48
+ float m1,
49
+ int n_head_log2
50
+ ) {
51
+ src0 = src0 + offset0;
52
+ src1 = src1 + offset1;
53
+ src2 = src2 + offset2;
54
+ dst = dst + offsetd;
55
+
56
+ int i03 = get_group_id(2);
57
+ int i02 = get_group_id(1);
58
+ int i01 = get_group_id(0);
59
+
60
+ int i13 = i03%ne13;
61
+ int i12 = i02%ne12;
62
+ int i11 = i01;
63
+
64
+ global float4 * psrc4 = (global float4 *)(src0 + i01*nb01 + i02*nb02 + i03*nb03);
65
+ global float4 * pmask = src1 != src0 ? (global float4 *)(src1 + i11*nb11 + i12*nb12 + i13*nb13) : 0;
66
+ global float * psrc2 = src2 != src0 ? (global float *)(src2) : 0;
67
+ global float4 * pdst4 = (global float4 *)(dst + i01*nb1 + i02*nb2 + i03*nb3);
68
+
69
+ float slope = 1.0f;
70
+
71
+ // ALiBi
72
+ if (max_bias > 0.0f) {
73
+ int h = i02;
74
+
75
+ float base = h < n_head_log2 ? m0 : m1;
76
+ int exp = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1;
77
+
78
+ slope = pow(base, exp);
79
+ }
80
+
81
+ // parallel max
82
+ float4 lmax4 = psrc2 ? psrc2[i02] : -INFINITY;
83
+ for (int i00 = get_local_id(0); i00 < ne00/4; i00 += get_local_size(0)) {
84
+ lmax4 = fmax(lmax4, psrc4[i00]*scale + (pmask ? slope*pmask[i00] : 0.0f));
85
+ }
86
+ float lmax = fmax(fmax(lmax4.s0, lmax4.s1), fmax(lmax4.s2, lmax4.s3));
87
+
88
+ const float max = sub_group_reduce_max(lmax);
89
+
90
+ // parallel sum
91
+ float4 lsum4 = 0.0f;
92
+ for (int i00 = get_local_id(0); i00 < ne00/4; i00 += get_local_size(0)) {
93
+ const float4 exp_psrc4 = exp((psrc4[i00]*scale + (pmask ? slope*pmask[i00] : 0.0f)) - max);
94
+ lsum4 += exp_psrc4;
95
+ pdst4[i00] = exp_psrc4;
96
+ }
97
+ float lsum = lsum4.s0 + lsum4.s1 + lsum4.s2 + lsum4.s3;
98
+
99
+ float sum = sub_group_reduce_add(lsum);
100
+
101
+ if (psrc2) {
102
+ sum += exp(psrc2[i02] - max);
103
+ }
104
+
105
+ for (int i00 = get_local_id(0); i00 < ne00/4; i00 += get_local_size(0)) {
106
+ pdst4[i00] /= sum;
107
+ }
108
+ }
llama.cpp/ggml/src/ggml-opencl/kernels/softmax_f16.cl ADDED
@@ -0,0 +1,107 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #pragma OPENCL EXTENSION cl_khr_fp16 : enable
2
+
3
+ #ifdef cl_intel_subgroups
4
+ #pragma OPENCL EXTENSION cl_intel_subgroups : enable
5
+ #else
6
+ #pragma OPENCL EXTENSION cl_khr_subgroups : enable
7
+ #endif
8
+
9
+ #ifdef cl_intel_required_subgroup_size
10
+ #pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
11
+ #define INTEL_GPU 1
12
+ #define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
13
+ #define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
14
+ #elif defined(cl_qcom_reqd_sub_group_size)
15
+ #pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
16
+ #define ADRENO_GPU 1
17
+ #define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
18
+ #define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
19
+ #endif
20
+
21
+ #ifdef ADRENO_GPU
22
+ REQD_SUBGROUP_SIZE_64
23
+ #endif
24
+ kernel void kernel_soft_max_f16(
25
+ global char * src0,
26
+ ulong offset0,
27
+ global char * src1,
28
+ ulong offset1,
29
+ global char * src2,
30
+ ulong offset2,
31
+ global char * dst,
32
+ ulong offsetd,
33
+ int ne00,
34
+ ulong nb01,
35
+ ulong nb02,
36
+ ulong nb03,
37
+ int ne12,
38
+ int ne13,
39
+ ulong nb11,
40
+ ulong nb12,
41
+ ulong nb13,
42
+ ulong nb1,
43
+ ulong nb2,
44
+ ulong nb3,
45
+ float scale,
46
+ float max_bias,
47
+ float m0,
48
+ float m1,
49
+ int n_head_log2
50
+ ) {
51
+ src0 = src0 + offset0;
52
+ src1 = src1 + offset1;
53
+ src2 = src2 + offset2;
54
+ dst = dst + offsetd;
55
+
56
+ int i03 = get_group_id(2);
57
+ int i02 = get_group_id(1);
58
+ int i01 = get_group_id(0);
59
+
60
+ int i13 = i03%ne13;
61
+ int i12 = i02%ne12;
62
+ int i11 = i01;
63
+
64
+ global float * psrc0 = (global float *)(src0 + i01*nb01 + i02*nb02 + i03*nb03);
65
+ global half * pmask = src1 != src0 ? (global half *)(src1 + i11*nb11 + i12*nb12 + i13*nb13) : 0;
66
+ global float * psrc2 = src2 != src0 ? (global float *)(src2) : 0;
67
+ global float * pdst = (global float *)(dst + i01*nb1 + i02*nb2 + i03*nb3);
68
+
69
+ float slope = 1.0f;
70
+
71
+ // ALiBi
72
+ if (max_bias > 0.0f) {
73
+ int h = i02;
74
+
75
+ float base = h < n_head_log2 ? m0 : m1;
76
+ int exp = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1;
77
+
78
+ slope = pow(base, exp);
79
+ }
80
+
81
+ // parallel max
82
+ float lmax = psrc2 ? psrc2[i02] : -INFINITY;
83
+ for (int i00 = get_local_id(0); i00 < ne00; i00 += get_local_size(0)) {
84
+ lmax = fmax(lmax, psrc0[i00]*scale + (pmask ? slope*pmask[i00] : 0.0f));
85
+ }
86
+ float max = sub_group_reduce_max(lmax);
87
+
88
+ // parallel sum
89
+ float lsum = 0.0f;
90
+ for (int i00 = get_local_id(0); i00 < ne00; i00 += get_local_size(0)) {
91
+ float exp_psrc0 = exp((psrc0[i00]*scale + (pmask ? slope*pmask[i00] : 0.0f)) - max);
92
+ lsum += exp_psrc0;
93
+ // Remember the result of exp here. exp is expensive, so we really do not
94
+ // wish to compute it twice.
95
+ pdst[i00] = exp_psrc0;
96
+ }
97
+
98
+ float sum = sub_group_reduce_add(lsum);
99
+
100
+ if (psrc2) {
101
+ sum += exp(psrc2[i02] - max);
102
+ }
103
+
104
+ for (int i00 = get_local_id(0); i00 < ne00; i00 += get_local_size(0)) {
105
+ pdst[i00] /= sum;
106
+ }
107
+ }
llama.cpp/ggml/src/ggml-opencl/kernels/softmax_f32.cl ADDED
@@ -0,0 +1,107 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #pragma OPENCL EXTENSION cl_khr_fp16 : enable
2
+
3
+ #ifdef cl_intel_subgroups
4
+ #pragma OPENCL EXTENSION cl_intel_subgroups : enable
5
+ #else
6
+ #pragma OPENCL EXTENSION cl_khr_subgroups : enable
7
+ #endif
8
+
9
+ #ifdef cl_intel_required_subgroup_size
10
+ #pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
11
+ #define INTEL_GPU 1
12
+ #define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
13
+ #define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
14
+ #elif defined(cl_qcom_reqd_sub_group_size)
15
+ #pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
16
+ #define ADRENO_GPU 1
17
+ #define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
18
+ #define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
19
+ #endif
20
+
21
+ #ifdef ADRENO_GPU
22
+ REQD_SUBGROUP_SIZE_64
23
+ #endif
24
+ kernel void kernel_soft_max(
25
+ global char * src0,
26
+ ulong offset0,
27
+ global char * src1,
28
+ ulong offset1,
29
+ global char * src2,
30
+ ulong offset2,
31
+ global char * dst,
32
+ ulong offsetd,
33
+ int ne00,
34
+ ulong nb01,
35
+ ulong nb02,
36
+ ulong nb03,
37
+ int ne12,
38
+ int ne13,
39
+ ulong nb11,
40
+ ulong nb12,
41
+ ulong nb13,
42
+ ulong nb1,
43
+ ulong nb2,
44
+ ulong nb3,
45
+ float scale,
46
+ float max_bias,
47
+ float m0,
48
+ float m1,
49
+ int n_head_log2
50
+ ) {
51
+ src0 = src0 + offset0;
52
+ src1 = src1 + offset1;
53
+ src2 = src2 + offset2;
54
+ dst = dst + offsetd;
55
+
56
+ int i03 = get_group_id(2);
57
+ int i02 = get_group_id(1);
58
+ int i01 = get_group_id(0);
59
+
60
+ int i13 = i03%ne13;
61
+ int i12 = i02%ne12;
62
+ int i11 = i01;
63
+
64
+ global float * psrc0 = (global float *)(src0 + i01*nb01 + i02*nb02 + i03*nb03);
65
+ global float * pmask = src1 != src0 ? (global float *)(src1 + i11*nb11 + i12*nb12 + i13*nb13) : 0;
66
+ global float * psrc2 = src2 != src0 ? (global float *)(src2) : 0;
67
+ global float * pdst = (global float *)(dst + i01*nb1 + i02*nb2 + i03*nb3);
68
+
69
+ float slope = 1.0f;
70
+
71
+ // ALiBi
72
+ if (max_bias > 0.0f) {
73
+ int h = i02;
74
+
75
+ float base = h < n_head_log2 ? m0 : m1;
76
+ int exp = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1;
77
+
78
+ slope = pow(base, exp);
79
+ }
80
+
81
+ // parallel max
82
+ float lmax = psrc2 ? psrc2[i02] : -INFINITY;
83
+ for (int i00 = get_local_id(0); i00 < ne00; i00 += get_local_size(0)) {
84
+ lmax = fmax(lmax, psrc0[i00]*scale + (pmask ? slope*pmask[i00] : 0.0f));
85
+ }
86
+ float max = sub_group_reduce_max(lmax);
87
+
88
+ // parallel sum
89
+ float lsum = 0.0f;
90
+ for (int i00 = get_local_id(0); i00 < ne00; i00 += get_local_size(0)) {
91
+ float exp_psrc0 = exp((psrc0[i00]*scale + (pmask ? slope*pmask[i00] : 0.0f)) - max);
92
+ lsum += exp_psrc0;
93
+ // Remember the result of exp here. exp is expensive, so we really do not
94
+ // wish to compute it twice.
95
+ pdst[i00] = exp_psrc0;
96
+ }
97
+
98
+ float sum = sub_group_reduce_add(lsum);
99
+
100
+ if (psrc2) {
101
+ sum += exp(psrc2[i02] - max);
102
+ }
103
+
104
+ for (int i00 = get_local_id(0); i00 < ne00; i00 += get_local_size(0)) {
105
+ pdst[i00] /= sum;
106
+ }
107
+ }
llama.cpp/ggml/src/ggml-opencl/kernels/softplus.cl ADDED
@@ -0,0 +1,116 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #pragma OPENCL EXTENSION cl_khr_fp16 : enable
2
+
3
+ //------------------------------------------------------------------------------
4
+ // softplus
5
+ //------------------------------------------------------------------------------
6
+
7
+ kernel void kernel_softplus_f32(
8
+ global const float * src0,
9
+ ulong offset0,
10
+ global float * dst,
11
+ ulong offsetd
12
+ ) {
13
+ src0 = (global float*)((global char*)src0 + offset0);
14
+ dst = (global float*)((global char*)dst + offsetd);
15
+
16
+ dst[get_global_id(0)] = (src0[get_global_id(0)] > 20.0f) ? src0[get_global_id(0)] : log(1.0f + exp(src0[get_global_id(0)]));
17
+ }
18
+
19
+ kernel void kernel_softplus_f32_4(
20
+ global const float4 * src0,
21
+ ulong offset0,
22
+ global float4 * dst,
23
+ ulong offsetd
24
+ ) {
25
+ src0 = (global float4*)((global char*)src0 + offset0);
26
+ dst = (global float4*)((global char*)dst + offsetd);
27
+
28
+ dst[get_global_id(0)] = (src0[get_global_id(0)] > 20.0f) ? src0[get_global_id(0)] : log(1.0f + exp(src0[get_global_id(0)]));
29
+ }
30
+
31
+ kernel void kernel_softplus_f16(
32
+ global const half * src0,
33
+ ulong offset0,
34
+ global half * dst,
35
+ ulong offsetd
36
+ ) {
37
+ src0 = (global half*)((global char*)src0 + offset0);
38
+ dst = (global half*)((global char*)dst + offsetd);
39
+
40
+ const float x = convert_float(src0[get_global_id(0)]);
41
+ dst[get_global_id(0)] = convert_half_rte((x > 20.0f) ? x : log(1.0f + exp(x)));
42
+ }
43
+
44
+ kernel void kernel_softplus_f16_4(
45
+ global const half4 * src0,
46
+ ulong offset0,
47
+ global half4 * dst,
48
+ ulong offsetd
49
+ ) {
50
+ src0 = (global half4*)((global char*)src0 + offset0);
51
+ dst = (global half4*)((global char*)dst + offsetd);
52
+
53
+ const float4 x = convert_float4(src0[get_global_id(0)]);
54
+ dst[get_global_id(0)] = convert_half4_rte((x > 20.0f) ? x : log(1.0f + exp(x)));
55
+ }
56
+
57
+ kernel void kernel_softplus_f32_nc(
58
+ global const char * src0,
59
+ ulong offset0,
60
+ global char * dst,
61
+ ulong offsetd,
62
+ int ne00,
63
+ ulong nb00,
64
+ ulong nb01,
65
+ ulong nb02,
66
+ ulong nb03,
67
+ ulong nb0,
68
+ ulong nb1,
69
+ ulong nb2,
70
+ ulong nb3
71
+ ) {
72
+ src0 = src0 + offset0;
73
+ dst = dst + offsetd;
74
+
75
+ const int i3 = get_group_id(2);
76
+ const int i2 = get_group_id(1);
77
+ const int i1 = get_group_id(0);
78
+
79
+ for (int i0 = get_local_id(0); i0 < ne00; i0 += get_local_size(0)) {
80
+ global const float * x = (global const float *)(src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);
81
+ global float * y = (global float *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
82
+
83
+ *y = (*x > 20.0f) ? *x : log(1.0f + exp(*x));
84
+ }
85
+ }
86
+
87
+ kernel void kernel_softplus_f16_nc(
88
+ global const char * src0,
89
+ ulong offset0,
90
+ global char * dst,
91
+ ulong offsetd,
92
+ int ne00,
93
+ ulong nb00,
94
+ ulong nb01,
95
+ ulong nb02,
96
+ ulong nb03,
97
+ ulong nb0,
98
+ ulong nb1,
99
+ ulong nb2,
100
+ ulong nb3
101
+ ) {
102
+ src0 = src0 + offset0;
103
+ dst = dst + offsetd;
104
+
105
+ const int i3 = get_group_id(2);
106
+ const int i2 = get_group_id(1);
107
+ const int i1 = get_group_id(0);
108
+
109
+ for (int i0 = get_local_id(0); i0 < ne00; i0 += get_local_size(0)) {
110
+ global const half * hx = (global const half *)(src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);
111
+ global half * hy = (global half *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
112
+
113
+ const float x = convert_float(*hx);
114
+ *hy = convert_half_rte((x > 20.0f) ? x : log(1.0f + exp(x)));
115
+ }
116
+ }
llama.cpp/ggml/src/ggml-opencl/kernels/solve_tri.cl ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #pragma OPENCL EXTENSION cl_khr_fp16 : enable
2
+
3
+ //------------------------------------------------------------------------------
4
+ // solve_tri
5
+ //------------------------------------------------------------------------------
6
+ kernel void kernel_solve_tri_f32(
7
+ global uchar * src0,
8
+ ulong offset0,
9
+ global uchar * src1,
10
+ ulong offset1,
11
+ global uchar * dst,
12
+ ulong offsetd,
13
+ int n,
14
+ int k,
15
+ ulong nb00,
16
+ ulong nb01,
17
+ ulong nb02,
18
+ ulong nb03,
19
+ ulong nb10,
20
+ ulong nb11,
21
+ ulong nb12,
22
+ ulong nb13,
23
+ ulong nb0,
24
+ ulong nb1,
25
+ ulong nb2,
26
+ ulong nb3
27
+ ) {
28
+ int col = get_global_id(0);
29
+ int i2 = get_global_id(1);
30
+ int i3 = get_global_id(2);
31
+
32
+ global const uchar * Lb = src0 + offset0 + i2 * nb02 + i3 * nb03;
33
+ global const uchar * Bb = src1 + offset1 + i2 * nb12 + i3 * nb13;
34
+ global uchar * Xb = dst + offsetd + i2 * nb2 + i3 * nb3;
35
+
36
+ for(int row = 0; row < n; ++row){
37
+ global const float *pB = (global const float *)(Bb + row * nb11 + col * nb10);
38
+
39
+ float sum = 0.0f;
40
+ for(int j = 0; j < row; ++j){
41
+ global const float *pL = (global const float *)(Lb + row * nb01 + j * nb00);
42
+ global const float *pX = (global const float *)(Xb + j * nb1 + col * nb0);
43
+ sum += (*pL) * (*pX);
44
+ }
45
+
46
+ global const float * pDiag = (global const float *)(Lb + row * nb01 + row *nb00);
47
+ global float * pOut = (global float *)(Xb + row * nb1 + col *nb0);
48
+
49
+ *pOut = ((* pB) - sum) / (*pDiag);
50
+ }
51
+ }
llama.cpp/ggml/src/ggml-opencl/kernels/sqr.cl ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #pragma OPENCL EXTENSION cl_khr_fp16 : enable
2
+
3
+ kernel void kernel_sqr_cont_f32(
4
+ global float * src0,
5
+ ulong offset0,
6
+ global float * dst,
7
+ ulong offsetd
8
+ ) {
9
+ src0 = (global float*)((global char*)src0 + offset0);
10
+ dst = (global float*)((global char*)dst + offsetd);
11
+
12
+ uint gid = get_global_id(0);
13
+ dst[gid] = src0[gid] * src0[gid];
14
+ }
15
+
16
+ kernel void kernel_sqr_cont_f32_4(
17
+ global float4 * src0,
18
+ ulong offset0,
19
+ global float4 * dst,
20
+ ulong offsetd
21
+ ) {
22
+ src0 = (global float4*)((global char*)src0 + offset0);
23
+ dst = (global float4*)((global char*)dst + offsetd);
24
+
25
+ uint gid = get_global_id(0);
26
+ dst[gid] = src0[gid] * src0[gid];
27
+ }
28
+
29
+ kernel void kernel_sqr_cont_f16(
30
+ global half * src0,
31
+ ulong offset0,
32
+ global half * dst,
33
+ ulong offsetd
34
+ ) {
35
+ src0 = (global half*)((global char*)src0 + offset0);
36
+ dst = (global half*)((global char*)dst + offsetd);
37
+
38
+ uint gid = get_global_id(0);
39
+ dst[gid] = src0[gid] * src0[gid];
40
+ }
41
+
42
+ kernel void kernel_sqr_cont_f16_4(
43
+ global half4 * src0,
44
+ ulong offset0,
45
+ global half4 * dst,
46
+ ulong offsetd
47
+ ) {
48
+ src0 = (global half4*)((global char*)src0 + offset0);
49
+ dst = (global half4*)((global char*)dst + offsetd);
50
+
51
+ uint gid = get_global_id(0);
52
+ dst[gid] = src0[gid] * src0[gid];
53
+ }
llama.cpp/ggml/src/ggml-opencl/kernels/sqrt.cl ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #pragma OPENCL EXTENSION cl_khr_fp16 : enable
2
+
3
+ kernel void kernel_sqrt_cont_f32(
4
+ global float * src0,
5
+ ulong offset0,
6
+ global float * dst,
7
+ ulong offsetd
8
+ ) {
9
+ src0 = (global float*)((global char*)src0 + offset0);
10
+ dst = (global float*)((global char*)dst + offsetd);
11
+
12
+ uint gid = get_global_id(0);
13
+ dst[gid] = sqrt(src0[gid]);
14
+ }
15
+
16
+ kernel void kernel_sqrt_cont_f32_4(
17
+ global float4 * src0,
18
+ ulong offset0,
19
+ global float4 * dst,
20
+ ulong offsetd
21
+ ) {
22
+ src0 = (global float4*)((global char*)src0 + offset0);
23
+ dst = (global float4*)((global char*)dst + offsetd);
24
+
25
+ uint gid = get_global_id(0);
26
+ dst[gid] = sqrt(src0[gid]);
27
+ }
28
+
29
+ kernel void kernel_sqrt_cont_f16(
30
+ global half * src0,
31
+ ulong offset0,
32
+ global half * dst,
33
+ ulong offsetd
34
+ ) {
35
+ src0 = (global half*)((global char*)src0 + offset0);
36
+ dst = (global half*)((global char*)dst + offsetd);
37
+
38
+ uint gid = get_global_id(0);
39
+ dst[gid] = convert_half(sqrt(convert_float(src0[gid])));
40
+ }
41
+
42
+ kernel void kernel_sqrt_cont_f16_4(
43
+ global half4 * src0,
44
+ ulong offset0,
45
+ global half4 * dst,
46
+ ulong offsetd
47
+ ) {
48
+ src0 = (global half4*)((global char*)src0 + offset0);
49
+ dst = (global half4*)((global char*)dst + offsetd);
50
+
51
+ uint gid = get_global_id(0);
52
+ dst[gid] = convert_half4(sqrt(convert_float4(src0[gid])));
53
+ }
llama.cpp/ggml/src/ggml-opencl/kernels/ssm_conv.cl ADDED
@@ -0,0 +1,77 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ kernel void kernel_ssm_conv_f32_f32(
2
+ global char * src0,
3
+ ulong offset0,
4
+ global char * src1,
5
+ ulong offset1,
6
+ global char * dst,
7
+ ulong offsetd,
8
+ ulong nb00,
9
+ ulong nb01,
10
+ ulong nb02,
11
+ int ne10,
12
+ ulong nb11,
13
+ ulong nb0,
14
+ ulong nb1,
15
+ ulong nb2
16
+ ){
17
+ src0 = src0 + offset0;
18
+ src1 = src1 + offset1;
19
+ dst = dst + offsetd;
20
+
21
+ int ir = get_global_id(0);
22
+ int i2 = get_global_id(1);
23
+ int i3 = get_global_id(2);
24
+
25
+ int nc = ne10;
26
+
27
+ global float * s = (global float *) (src0 + ir*nb01 + i2*nb00 + i3*nb02);
28
+ global float * c = (global float *) (src1 + ir*nb11);
29
+ global float * d = (global float *) (dst + ir*nb0 + i2*nb1 + i3*nb2);
30
+
31
+ float sumf = 0.0f;
32
+
33
+ for (int i0 = 0; i0 < nc; ++i0) {
34
+ sumf += s[i0] * c[i0];
35
+ }
36
+
37
+ d[0] = sumf;
38
+ }
39
+
40
+ kernel void kernel_ssm_conv_f32_f32_4(
41
+ global char * src0,
42
+ ulong offset0,
43
+ global char * src1,
44
+ ulong offset1,
45
+ global char * dst,
46
+ ulong offsetd,
47
+ ulong nb00,
48
+ ulong nb01,
49
+ ulong nb02,
50
+ int ne10,
51
+ ulong nb11,
52
+ ulong nb0,
53
+ ulong nb1,
54
+ ulong nb2
55
+ ) {
56
+ src0 = src0 + offset0;
57
+ src1 = src1 + offset1;
58
+ dst = dst + offsetd;
59
+
60
+ int ir = get_global_id(0);
61
+ int i2 = get_global_id(1);
62
+ int i3 = get_global_id(2);
63
+
64
+ int nc = ne10;
65
+
66
+ global float4 * s = (global float4 *) (src0 + ir*nb01 + i2*nb00 + i3*nb02);
67
+ global float4 * c = (global float4 *) (src1 + ir*nb11);
68
+ global float * d = (global float *) (dst + ir*nb0 + i2*nb1 + i3*nb2);
69
+
70
+ float sumf = 0.0f;
71
+
72
+ for (int i0 = 0; i0 < nc/4; ++i0) {
73
+ sumf += dot(s[i0], c[i0]);
74
+ }
75
+
76
+ d[0] = sumf;
77
+ }
llama.cpp/ggml/src/ggml-opencl/kernels/sub.cl ADDED
@@ -0,0 +1,138 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #pragma OPENCL EXTENSION cl_khr_fp16 : enable
2
+
3
+ //------------------------------------------------------------------------------
4
+ // div
5
+ //------------------------------------------------------------------------------
6
+ kernel void kernel_sub(
7
+ global char * src0,
8
+ ulong offset0,
9
+ global char * src1,
10
+ ulong offset1,
11
+ global char * dst,
12
+ ulong offsetd,
13
+ ulong nb00,
14
+ ulong nb01,
15
+ ulong nb02,
16
+ ulong nb03,
17
+ int ne10,
18
+ int ne11,
19
+ int ne12,
20
+ int ne13,
21
+ ulong nb10,
22
+ ulong nb11,
23
+ ulong nb12,
24
+ ulong nb13,
25
+ int ne0,
26
+ ulong nb0,
27
+ ulong nb1,
28
+ ulong nb2,
29
+ ulong nb3
30
+ ) {
31
+ src0 = src0 + offset0;
32
+ src1 = src1 + offset1;
33
+ dst = dst + offsetd;
34
+
35
+ int i03 = get_group_id(2);
36
+ int i02 = get_group_id(1);
37
+ int i01 = get_group_id(0);
38
+
39
+ int i13 = i03 % ne13;
40
+ int i12 = i02 % ne12;
41
+ int i11 = i01 % ne11;
42
+
43
+ global char * src0_ptr = src0 + i03*nb03 + i02*nb02 + i01*nb01;
44
+ global char * src1_ptr = src1 + i13*nb13 + i12*nb12 + i11*nb11;
45
+ global char * dst_ptr = dst + i03*nb3 + i02*nb2 + i01*nb1;
46
+
47
+ for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) {
48
+ const int i10 = i0 % ne10;
49
+ *((global float *)(dst_ptr + i0*nb0)) = *((global float *)(src0_ptr + i0*nb00)) - *((global float *)(src1_ptr + i10*nb10));
50
+ }
51
+ }
52
+
53
+ // assumption: src1 is a row
54
+ // broadcast src1 into src0
55
+ kernel void kernel_sub_row(
56
+ global float4 * src0,
57
+ ulong offset0,
58
+ global float4 * src1,
59
+ ulong offset1,
60
+ global float4 * dst,
61
+ ulong offsetd,
62
+ int ne
63
+ ) {
64
+ src0 = (global float4*)((global char*)src0 + offset0);
65
+ src1 = (global float4*)((global char*)src1 + offset1);
66
+ dst = (global float4*)((global char*)dst + offsetd);
67
+
68
+ // This performs better than using %.
69
+ uint gid = get_global_id(0);
70
+ uint idx1 = gid - (gid/ne)*ne; // get_global_id(0) % ne
71
+ dst[gid] = src0[gid] - src1[idx1];
72
+ }
73
+
74
+ kernel void kernel_sub_f16(
75
+ global char * src0,
76
+ ulong offset0,
77
+ global char * src1,
78
+ ulong offset1,
79
+ global char * dst,
80
+ ulong offsetd,
81
+ ulong nb00,
82
+ ulong nb01,
83
+ ulong nb02,
84
+ ulong nb03,
85
+ int ne10,
86
+ int ne11,
87
+ int ne12,
88
+ int ne13,
89
+ ulong nb10,
90
+ ulong nb11,
91
+ ulong nb12,
92
+ ulong nb13,
93
+ int ne0,
94
+ ulong nb0,
95
+ ulong nb1,
96
+ ulong nb2,
97
+ ulong nb3
98
+ ) {
99
+ src0 = src0 + offset0;
100
+ src1 = src1 + offset1;
101
+ dst = dst + offsetd;
102
+
103
+ int i03 = get_group_id(2);
104
+ int i02 = get_group_id(1);
105
+ int i01 = get_group_id(0);
106
+
107
+ int i13 = i03 % ne13;
108
+ int i12 = i02 % ne12;
109
+ int i11 = i01 % ne11;
110
+
111
+ global char * src0_ptr = src0 + i03*nb03 + i02*nb02 + i01*nb01;
112
+ global char * src1_ptr = src1 + i13*nb13 + i12*nb12 + i11*nb11;
113
+ global char * dst_ptr = dst + i03*nb3 + i02*nb2 + i01*nb1;
114
+
115
+ for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) {
116
+ const int i10 = i0 % ne10;
117
+ *((global half *)(dst_ptr + i0*nb0)) = *((global half *)(src0_ptr + i0*nb00)) - *((global half *)(src1_ptr + i10*nb10));
118
+ }
119
+ }
120
+
121
+ kernel void kernel_sub_row_f16(
122
+ global half4 * src0,
123
+ ulong offset0,
124
+ global half4 * src1,
125
+ ulong offset1,
126
+ global half4 * dst,
127
+ ulong offsetd,
128
+ int ne
129
+ ) {
130
+ src0 = (global half4*)((global char*)src0 + offset0);
131
+ src1 = (global half4*)((global char*)src1 + offset1);
132
+ dst = (global half4*)((global char*)dst + offsetd);
133
+
134
+ // This performs better than using %.
135
+ uint gid = get_global_id(0);
136
+ uint idx1 = gid - (gid/ne)*ne; // get_global_id(0) % ne
137
+ dst[gid] = src0[gid] - src1[idx1];
138
+ }
llama.cpp/ggml/src/ggml-opencl/kernels/sum_rows.cl ADDED
@@ -0,0 +1,140 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #pragma OPENCL EXTENSION cl_khr_fp16 : enable
2
+ #pragma OPENCL EXTENSION cl_khr_subgroups : enable
3
+
4
+ // Most devices have max workgroup size of 1024, so this is enough for subgroup
5
+ // sizes of 16, 32, 64 and 128. Increase this value for smaller subgroups sizes
6
+ #define MAX_SUBGROUPS 64
7
+ kernel void kernel_sum_rows_f32(
8
+ global char * src0,
9
+ ulong offset0,
10
+ global char * dst,
11
+ ulong offsetd,
12
+ int ne00,
13
+ int ne01,
14
+ int ne02,
15
+ int ne03,
16
+ ulong nb01,
17
+ ulong nb02,
18
+ ulong nb03,
19
+ ulong nb1,
20
+ ulong nb2,
21
+ ulong nb3
22
+ ) {
23
+ src0 = src0 + offset0;
24
+ dst = dst + offsetd;
25
+
26
+ const int i3 = get_group_id(2);
27
+ const int i2 = get_group_id(1);
28
+ const int i1 = get_group_id(0);
29
+
30
+ const int lid = get_local_id(0);
31
+ const int lsize = get_local_size(0);
32
+
33
+ const uint sg_size = get_sub_group_size();
34
+ const uint sg_id = get_sub_group_id();
35
+ const uint sg_lid = get_sub_group_local_id();
36
+
37
+ __local float lmem[MAX_SUBGROUPS];
38
+
39
+ if (i3 >= ne03 || i2 >= ne02 || i1 >= ne01) {
40
+ return;
41
+ }
42
+
43
+ if(sg_id == 0){
44
+ lmem[sg_lid] = 0.0f;
45
+ }
46
+
47
+ global float * src_row = (global float *) (src0 + i1*nb01 + i2*nb02 + i3*nb03);
48
+ global float * dst_row = (global float *) (dst + i1*nb1 + i2*nb2 + i3*nb3);
49
+
50
+ float sumf = 0.0f;
51
+
52
+ for (int i0 = lid; i0 < ne00; i0 += lsize) {
53
+ sumf += src_row[i0];
54
+ }
55
+
56
+ sumf = sub_group_reduce_add(sumf);
57
+
58
+ barrier(CLK_LOCAL_MEM_FENCE);
59
+
60
+ if(sg_lid == 0){
61
+ lmem[sg_id] = sumf;
62
+ }
63
+
64
+ barrier(CLK_LOCAL_MEM_FENCE);
65
+
66
+ sumf = lmem[sg_lid];
67
+ sumf = sub_group_reduce_add(sumf);
68
+
69
+ if (lid == 0) {
70
+ dst_row[0] = sumf;
71
+ }
72
+ }
73
+
74
+ kernel void kernel_sum_rows_f32_4(
75
+ global char * src0,
76
+ ulong offset0,
77
+ global char * dst,
78
+ ulong offsetd,
79
+ int ne00,
80
+ int ne01,
81
+ int ne02,
82
+ int ne03,
83
+ ulong nb01,
84
+ ulong nb02,
85
+ ulong nb03,
86
+ ulong nb1,
87
+ ulong nb2,
88
+ ulong nb3
89
+ ) {
90
+ src0 = src0 + offset0;
91
+ dst = dst + offsetd;
92
+
93
+ const int i3 = get_group_id(2);
94
+ const int i2 = get_group_id(1);
95
+ const int i1 = get_group_id(0);
96
+
97
+ const int lid = get_local_id(0);
98
+ const int lsize = get_local_size(0);
99
+
100
+ const uint sg_size = get_sub_group_size();
101
+ const uint sg_id = get_sub_group_id();
102
+ const uint sg_lid = get_sub_group_local_id();
103
+
104
+ __local float lmem[MAX_SUBGROUPS];
105
+
106
+ if (i3 >= ne03 || i2 >= ne02 || i1 >= ne01) {
107
+ return;
108
+ }
109
+
110
+ if(sg_id == 0){
111
+ lmem[sg_lid] = 0.0f;
112
+ }
113
+
114
+ global float4 * src_row = (global float4 *) (src0 + i1*nb01 + i2*nb02 + i3*nb03);
115
+ global float * dst_row = (global float *) (dst + i1*nb1 + i2*nb2 + i3*nb3);
116
+
117
+ float4 sum_vec = (float4)0.0f;
118
+
119
+ for (int i0 = lid; i0 < ne00 / 4; i0 += lsize) {
120
+ sum_vec += src_row[i0];
121
+ }
122
+
123
+ float sumf = dot(sum_vec, (float4)(1.0f));
124
+ sumf = sub_group_reduce_add(sumf);
125
+
126
+ barrier(CLK_LOCAL_MEM_FENCE);
127
+
128
+ if(sg_lid == 0){
129
+ lmem[sg_id] = sumf;
130
+ }
131
+
132
+ barrier(CLK_LOCAL_MEM_FENCE);
133
+
134
+ sumf = lmem[sg_lid];
135
+ sumf = sub_group_reduce_add(sumf);
136
+
137
+ if (lid == 0) {
138
+ dst_row[0] = sumf;
139
+ }
140
+ }
llama.cpp/ggml/src/ggml-opencl/kernels/tanh.cl ADDED
@@ -0,0 +1,109 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #pragma OPENCL EXTENSION cl_khr_fp16 : enable
2
+
3
+ kernel void kernel_tanh_f32(
4
+ global const float * src0,
5
+ ulong offset0,
6
+ global float * dst,
7
+ ulong offsetd
8
+ ) {
9
+ src0 = (global float*)((global char*)src0 + offset0);
10
+ dst = (global float*)((global char*)dst + offsetd);
11
+
12
+ dst[get_global_id(0)] = tanh(src0[get_global_id(0)]);
13
+ }
14
+
15
+ kernel void kernel_tanh_f32_4(
16
+ global const float4 * src0,
17
+ ulong offset0,
18
+ global float4 * dst,
19
+ ulong offsetd
20
+ ) {
21
+ src0 = (global float4*)((global char*)src0 + offset0);
22
+ dst = (global float4*)((global char*)dst + offsetd);
23
+
24
+ dst[get_global_id(0)] = tanh(src0[get_global_id(0)]);
25
+ }
26
+
27
+ kernel void kernel_tanh_f16(
28
+ global const half * src0,
29
+ ulong offset0,
30
+ global half * dst,
31
+ ulong offsetd
32
+ ) {
33
+ src0 = (global half*)((global char*)src0 + offset0);
34
+ dst = (global half*)((global char*)dst + offsetd);
35
+
36
+ dst[get_global_id(0)] = tanh(src0[get_global_id(0)]);
37
+ }
38
+
39
+ kernel void kernel_tanh_f16_4(
40
+ global const half4 * src0,
41
+ ulong offset0,
42
+ global half4 * dst,
43
+ ulong offsetd
44
+ ) {
45
+ src0 = (global half4*)((global char*)src0 + offset0);
46
+ dst = (global half4*)((global char*)dst + offsetd);
47
+
48
+ dst[get_global_id(0)] = tanh(src0[get_global_id(0)]);
49
+ }
50
+
51
+ kernel void kernel_tanh_f32_nc(
52
+ global const char * src0,
53
+ ulong offset0,
54
+ global char * dst,
55
+ ulong offsetd,
56
+ int ne00,
57
+ ulong nb00,
58
+ ulong nb01,
59
+ ulong nb02,
60
+ ulong nb03,
61
+ ulong nb0,
62
+ ulong nb1,
63
+ ulong nb2,
64
+ ulong nb3
65
+ ) {
66
+ src0 = src0 + offset0;
67
+ dst = dst + offsetd;
68
+
69
+ const int i3 = get_group_id(2);
70
+ const int i2 = get_group_id(1);
71
+ const int i1 = get_group_id(0);
72
+
73
+ for (int i0 = get_local_id(0); i0 < ne00; i0 += get_local_size(0)) {
74
+ global const float * x = (global const float *)(src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);
75
+ global float * y = (global float *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
76
+
77
+ *y = tanh(*x);
78
+ }
79
+ }
80
+
81
+ kernel void kernel_tanh_f16_nc(
82
+ global const char * src0,
83
+ ulong offset0,
84
+ global char * dst,
85
+ ulong offsetd,
86
+ int ne00,
87
+ ulong nb00,
88
+ ulong nb01,
89
+ ulong nb02,
90
+ ulong nb03,
91
+ ulong nb0,
92
+ ulong nb1,
93
+ ulong nb2,
94
+ ulong nb3
95
+ ) {
96
+ src0 = src0 + offset0;
97
+ dst = dst + offsetd;
98
+
99
+ const int i3 = get_group_id(2);
100
+ const int i2 = get_group_id(1);
101
+ const int i1 = get_group_id(0);
102
+
103
+ for (int i0 = get_local_id(0); i0 < ne00; i0 += get_local_size(0)) {
104
+ global const half * x = (global const half *)(src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);
105
+ global half * y = (global half *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
106
+
107
+ *y = tanh(*x);
108
+ }
109
+ }
llama.cpp/ggml/src/ggml-opencl/kernels/transpose.cl ADDED
@@ -0,0 +1,143 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #pragma OPENCL EXTENSION cl_khr_fp16 : enable
2
+
3
+ // 16-bit transpose, loading/storing a 4x4 tile of elements
4
+ kernel void kernel_transpose_16(
5
+ __read_only image1d_buffer_t input,
6
+ __write_only image1d_buffer_t output,
7
+ const uint rows,
8
+ const uint cols
9
+ ) {
10
+
11
+ const int i = get_global_id(0);
12
+ const int j = get_global_id(1);
13
+ const int i_2 = i<<2;
14
+ const int j_2 = j<<2;
15
+
16
+ half4 temp0 = read_imageh(input, (j_2+0)*cols+i);
17
+ half4 temp1 = read_imageh(input, (j_2+1)*cols+i);
18
+ half4 temp2 = read_imageh(input, (j_2+2)*cols+i);
19
+ half4 temp3 = read_imageh(input, (j_2+3)*cols+i);
20
+
21
+ write_imageh(output, (i_2+0)*rows+j, (half4)(temp0.s0, temp1.s0, temp2.s0, temp3.s0));
22
+ write_imageh(output, (i_2+1)*rows+j, (half4)(temp0.s1, temp1.s1, temp2.s1, temp3.s1));
23
+ write_imageh(output, (i_2+2)*rows+j, (half4)(temp0.s2, temp1.s2, temp2.s2, temp3.s2));
24
+ write_imageh(output, (i_2+3)*rows+j, (half4)(temp0.s3, temp1.s3, temp2.s3, temp3.s3));
25
+ }
26
+
27
+ // Padded kernel for irregular shape
28
+ kernel void kernel_transpose_16_4x1(
29
+ __read_only image1d_buffer_t input,
30
+ __write_only image1d_buffer_t output,
31
+ const uint rows,
32
+ const uint cols
33
+ ) {
34
+
35
+ const int i = get_global_id(0);
36
+ const int j = get_global_id(1);
37
+ const int j_2 = j << 2;
38
+
39
+ half temp0 = read_imageh(input, (j_2 + 0) * cols + i).x;
40
+ half temp1 = read_imageh(input, (j_2 + 1) * cols + i).x;
41
+ half temp2 = read_imageh(input, (j_2 + 2) * cols + i).x;
42
+ half temp3 = read_imageh(input, (j_2 + 3) * cols + i).x;
43
+
44
+ write_imageh(output, i * rows + j, (half4)(temp0, temp1, temp2, temp3));
45
+ }
46
+
47
+ // Transpose treating each element as 8-bit using buffer
48
+ kernel void kernel_transpose_8_buf(
49
+ global const uchar * input,
50
+ global uchar * output,
51
+ const int ldi,
52
+ const int ldo
53
+ ) {
54
+ const int x = get_global_id(0);
55
+ const int y = get_global_id(1);
56
+
57
+ output[x*ldo + y] = input[y*ldi + x];
58
+ }
59
+
60
+ // Transpose treating each element as 16-bit using buffer
61
+ kernel void kernel_transpose_16_buf(
62
+ global const ushort * input,
63
+ global ushort * output,
64
+ const int ldi,
65
+ const int ldo
66
+ ) {
67
+ const int x = get_global_id(0);
68
+ const int y = get_global_id(1);
69
+
70
+ output[x*ldo + y] = input[y*ldi + x];
71
+ }
72
+
73
+ // Transpose treating each element as 32-bit using buffer
74
+ kernel void kernel_transpose_32_buf(
75
+ global const uint * input,
76
+ global uint * output,
77
+ const int ldi,
78
+ const int ldo
79
+ ) {
80
+ const int x = get_global_id(0);
81
+ const int y = get_global_id(1);
82
+
83
+ output[x*ldo + y] = input[y*ldi + x];
84
+ }
85
+
86
+ // 32-bit transpose, loading/storing a 4x4 tile of elements
87
+ kernel void kernel_transpose_32(
88
+ __read_only image1d_buffer_t input,
89
+ __write_only image1d_buffer_t output,
90
+ const uint rows,
91
+ const uint cols
92
+ ) {
93
+
94
+ const int i = get_global_id(0);
95
+ const int j = get_global_id(1);
96
+ const int i_2 = i<<2;
97
+ const int j_2 = j<<2;
98
+
99
+ float4 temp0 = read_imagef(input, (j_2+0)*cols+i);
100
+ float4 temp1 = read_imagef(input, (j_2+1)*cols+i);
101
+ float4 temp2 = read_imagef(input, (j_2+2)*cols+i);
102
+ float4 temp3 = read_imagef(input, (j_2+3)*cols+i);
103
+
104
+ write_imagef(output, (i_2+0)*rows+j, (float4)(temp0.s0, temp1.s0, temp2.s0, temp3.s0));
105
+ write_imagef(output, (i_2+1)*rows+j, (float4)(temp0.s1, temp1.s1, temp2.s1, temp3.s1));
106
+ write_imagef(output, (i_2+2)*rows+j, (float4)(temp0.s2, temp1.s2, temp2.s2, temp3.s2));
107
+ write_imagef(output, (i_2+3)*rows+j, (float4)(temp0.s3, temp1.s3, temp2.s3, temp3.s3));
108
+
109
+ }
110
+
111
+ // 32-bit transpose, loading/storing a 4x4 tile of elements
112
+ // Only used for activations
113
+ // converts to FP16
114
+ // also adds zero padding for non multiple of 8 prompt lengths
115
+ kernel void kernel_transpose_32_16(__read_only image1d_buffer_t input, __write_only image1d_buffer_t output, const uint rows, const uint cols, const uint padded_rows) {
116
+
117
+ const int i = get_global_id(0);
118
+ const int j = get_global_id(1);
119
+ const int i_2 = i<<2;
120
+ const int j_2 = j<<2;
121
+ half4 temp0 = {0,0,0,0}; // initialize outputs to 0
122
+ half4 temp1 = {0,0,0,0};
123
+ half4 temp2 = {0,0,0,0};
124
+ half4 temp3 = {0,0,0,0};
125
+
126
+ if((j_2+0)*cols+i*4+3 < rows*cols*16){ // only load from a valid location. Otherwise keep register data as 0
127
+ temp0 = read_imageh(input, (j_2+0)*cols+i);
128
+ }
129
+ if((j_2+1)*cols+i*4+3 < rows*cols*16){
130
+ temp1 = read_imageh(input, (j_2+1)*cols+i);
131
+ }
132
+ if((j_2+2)*cols+i*4+3 < rows*cols*16){
133
+ temp2 = read_imageh(input, (j_2+2)*cols+i);
134
+ }
135
+ if((j_2+3)*cols+i*4+3 < rows*cols*16){
136
+ temp3 = read_imageh(input, (j_2+3)*cols+i);
137
+ }
138
+
139
+ write_imageh(output, (i_2+0)*padded_rows+j, (half4)(temp0.s0, temp1.s0, temp2.s0, temp3.s0)); // no conditionals for output, includes zero padding
140
+ write_imageh(output, (i_2+1)*padded_rows+j, (half4)(temp0.s1, temp1.s1, temp2.s1, temp3.s1));
141
+ write_imageh(output, (i_2+2)*padded_rows+j, (half4)(temp0.s2, temp1.s2, temp2.s2, temp3.s2));
142
+ write_imageh(output, (i_2+3)*padded_rows+j, (half4)(temp0.s3, temp1.s3, temp2.s3, temp3.s3));
143
+ }
llama.cpp/ggml/src/ggml-opencl/kernels/tri.cl ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #pragma OPENCL EXTENSION cl_khr_fp16 : enable
2
+
3
+ //------------------------------------------------------------------------------
4
+ // tri
5
+ //------------------------------------------------------------------------------
6
+ __kernel void kernel_tri_f32(
7
+ global float * src0,
8
+ ulong offset0,
9
+ global float * dst,
10
+ ulong offsetd,
11
+ int n,
12
+ int ne0,
13
+ int ne1,
14
+ int tri_type
15
+ ) {
16
+ src0 = (global float*)((global char*)src0 + offset0);
17
+ dst = (global float*)((global char*)dst + offsetd);
18
+
19
+ int idx = get_global_id(0);
20
+ if (idx >= n) return;
21
+
22
+ int i0 = idx % ne0;
23
+ int i1 = (idx / ne0) % ne1;
24
+
25
+ int keep = 0;
26
+ if (tri_type == 0) keep = (i0 >= i1);
27
+ else if (tri_type == 1) keep = (i0 > i1);
28
+ else if (tri_type == 2) keep = (i0 <= i1);
29
+ else keep = (i0 < i1);
30
+
31
+ dst[idx] = keep ? src0[idx] : 0.0f;
32
+ }