darthrubik/llama-cpp
1
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases Wiki Activity

opencl: add dp4a GEMM variants to vector load activation (#30753)
Copilot Setup Steps / copilot-setup-steps (push) Has been cancelled
Check Pre-Tokenizer Hashes / pre-tokenizer-hashes (push) Has been cancelled
Python check requirements.txt / check-requirements (push) Has been cancelled
Python Type-Check / python type-check (push) Has been cancelled
Update Operations Documentation / update-ops-docs (push) Has been cancelled

Browse Source 
* opencl: add dp4a GEMM variants with vector activation loads

* opencl: rename kernels and env vars
This commit is contained in:
lhez
2026-10-21 00:48:13 -07:00
committed by GitHub
parent 74b115ca6e
commit b531e4c749
6 changed files with 679 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ggml/src/ggml-opencl/ggml-opencl.cpp
+41 -9
View File
@@ -1243,6 +1243,7 @@ struct ggml_backend_opencl_context {
cl_kernel kernel_gemm_noshuffle_q4_1_f32;
cl_kernel kernel_gemm_noshuffle_q8_0_f32, kernel_gemm_noshuffle_q8_0_f32_bin;
cl_kernel kernel_gemm_noshuffle_q8_0_q8_1_dp4a = nullptr; // dp4a (int8) dense q8_0 prefill GEMM (opt-in)
cl_kernel kernel_gemm_noshuffle_q8_0_q8_1_dp4a_act_vload4 = nullptr;
cl_kernel kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg = nullptr; // q8_0 dense dp4a, weights via texture (opt-in)
cl_kernel kernel_gemm_noshuffle_q8_0_q8_1_dp4a_ila_a8_bin = nullptr;
cl_kernel kernel_gemv_noshuffle_q8_0_f32;
@@ -1264,6 +1265,7 @@ struct ggml_backend_opencl_context {
cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a_ila_a8_bin;
cl_kernel kernel_gemv_noshuffle_q4_k_f32_32b_trans;
cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a = nullptr; // dp4a (int8) dense prefill GEMM
cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a_act_vload4 = nullptr;
cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg = nullptr; // dp4a dense prefill GEMM, weights via texture (X1 opt-in)
cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a_narrow = nullptr;
cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a_narrow_wimg = nullptr;
@@ -1271,7 +1273,9 @@ struct ggml_backend_opencl_context {
int tile_n_dp4a_narrow_q4_k = 32;
int max_n_dp4a_narrow_q4_k = 16;
cl_kernel kernel_gemm_noshuffle_q5_k_q8_1_dp4a = nullptr; // dp4a (int8) dense q5_K prefill GEMM
cl_kernel kernel_gemm_noshuffle_q5_k_q8_1_dp4a_act_vload4 = nullptr;
cl_kernel kernel_gemm_noshuffle_q6_k_q8_1_dp4a = nullptr; // dp4a (int8) dense q6_K prefill GEMM
cl_kernel kernel_gemm_noshuffle_q6_k_q8_1_dp4a_act_vload4 = nullptr;
cl_kernel kernel_quant_a_q8_1; // plain activation q8_1 pre-pass
cl_kernel kernel_gemm_noshuffle_q4_k_f32_r1;
cl_kernel kernel_gemm_noshuffle_q4_k_f32_kimg;
@@ -1305,6 +1309,7 @@ struct ggml_backend_opencl_context {
cl_kernel kernel_gemm_noshuffle_iq4_nl_f32;
cl_kernel kernel_gemm_noshuffle_iq4_nl_q8_1_dp4a = nullptr; // dp4a (int8) dense IQ4_NL prefill GEMM
cl_kernel kernel_gemm_noshuffle_q4_0_q8_1_dp4a = nullptr; // dp4a (int8) dense q4_0 prefill GEMM
cl_kernel kernel_gemm_noshuffle_q4_0_q8_1_dp4a_act_vload4 = nullptr;
cl_kernel kernel_gemm_coopk_q4_0_q8_1_dp4a_c4 = nullptr;
cl_kernel kernel_gemm_coopk_q4_0_q8_1_dp4a_c2 = nullptr;
int nsg_coopk_dp4a_q4_0 = 8;
@@ -4347,6 +4352,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
#endif
cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);
CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_0_q8_1_dp4a", &err), err));
CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a_act_vload4 = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_0_q8_1_dp4a_act_vload4", &err), err));
CL_CHECK(clReleaseProgram(prog));
GGML_LOG_CONT(".");
}
@@ -4486,6 +4492,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), dp4a_opts);
CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_k_q8_1_dp4a", &err), err));
CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg", &err), err));
CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_act_vload4 = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_k_q8_1_dp4a_act_vload4", &err), err));
CL_CHECK(clReleaseProgram(prog));
// Compile a smaller N tile to avoid unused columns in small batches.
@@ -4518,6 +4525,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);
CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q8_0_q8_1_dp4a", &err), err));
CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg = clCreateKernel(prog, "kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg", &err), err));
CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_act_vload4 = clCreateKernel(prog, "kernel_gemm_noshuffle_q8_0_q8_1_dp4a_act_vload4", &err), err));
CL_CHECK(clReleaseProgram(prog));
GGML_LOG_CONT(".");
}
@@ -4533,6 +4541,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
#endif
cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);
CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_k_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q5_k_q8_1_dp4a", &err), err));
CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_k_q8_1_dp4a_act_vload4 = clCreateKernel(prog, "kernel_gemm_noshuffle_q5_k_q8_1_dp4a_act_vload4", &err), err));
CL_CHECK(clReleaseProgram(prog));
GGML_LOG_CONT(".");
}
@@ -4548,6 +4557,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
#endif
cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);
CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q6_k_q8_1_dp4a", &err), err));
CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a_act_vload4 = clCreateKernel(prog, "kernel_gemm_noshuffle_q6_k_q8_1_dp4a_act_vload4", &err), err));
CL_CHECK(clReleaseProgram(prog));
GGML_LOG_CONT(".");
}
@@ -20430,11 +20440,13 @@ static void ggml_cl_mul_mat_q4_0_f32_adreno(ggml_backend_t backend, const ggml_t
return;
}
// dp4a (int8) dense prefill GEMM, default off
static const char * q4_0_act_vload4_env = getenv("GGML_OPENCL_Q4_0_DP4A_ACT_VLOAD4");
const bool q4_0_act_vload4_on = q4_0_act_vload4_env == nullptr || atoi(q4_0_act_vload4_env) != 0;
// Enable dense prefill by default on X2E when the uint4 activation tile is enabled.
static const char * q4_0_dense_dp4a_env = getenv("GGML_OPENCL_Q4_0_DENSE_DP4A");
bool q4_0_dense_dp4a_on = q4_0_dense_dp4a_env
? (atoi(q4_0_dense_dp4a_env) != 0)
: false;
: (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E && q4_0_act_vload4_on);
// dot prod has to be available
q4_0_dense_dp4a_on = backend_ctx->has_integer_dot && q4_0_dense_dp4a_on;
@@ -20461,7 +20473,9 @@ static void ggml_cl_mul_mat_q4_0_f32_adreno(ggml_backend_t backend, const ggml_t
size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) };
backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst);
cl_kernel dk = backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a;
cl_kernel dk = q4_0_act_vload4_on
? backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a_act_vload4
: backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a;
int ai = 0;
CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q4_0->q));
CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q4_0->d));
@@ -21754,8 +21768,13 @@ static void ggml_cl_mul_mat_q8_0_f32_adreno(ggml_backend_t backend, const ggml_t
}
}
cl_kernel dk = use_wimg ? backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg
: backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a;
static const char * q8_0_act_vload4_env = getenv("GGML_OPENCL_Q8_0_DP4A_ACT_VLOAD4");
const bool q8_0_act_vload4_on = q8_0_act_vload4_env == nullptr || atoi(q8_0_act_vload4_env) != 0;
cl_kernel dk = use_wimg
? backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg
: q8_0_act_vload4_on
? backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_act_vload4
: backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a;
int ai = 0;
if (use_wimg) {
CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &q8_q_img));
@@ -22703,11 +22722,16 @@ static void ggml_cl_mul_mat_q4_k_f32_adreno(ggml_backend_t backend, const ggml_t
const bool use_narrow = backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_narrow != nullptr
&& N <= backend_ctx->max_n_dp4a_narrow_q4_k;
static const char * q4_k_act_vload4_env = getenv("GGML_OPENCL_Q4_K_DP4A_ACT_VLOAD4");
const bool q4_k_act_vload4_on = q4_k_act_vload4_env == nullptr || atoi(q4_k_act_vload4_env) != 0;
cl_kernel dk = use_narrow
? (use_wimg ? backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_narrow_wimg
: backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_narrow)
: (use_wimg ? backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg
: backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a);
: use_wimg
? backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg
: q4_k_act_vload4_on
? backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_act_vload4
: backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a;
int ai = 0;
if (use_wimg) {
CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &q4k_q_img));
@@ -23369,7 +23393,11 @@ static void ggml_cl_mul_mat_q6_K_f32_adreno(ggml_backend_t backend, const ggml_t
size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) };
backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst);
cl_kernel dk = backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a;
static const char * q6_k_act_vload4_env = getenv("GGML_OPENCL_Q6_K_DP4A_ACT_VLOAD4");
const bool q6_k_act_vload4_on = q6_k_act_vload4_env == nullptr || atoi(q6_k_act_vload4_env) != 0;
cl_kernel dk = q6_k_act_vload4_on
? backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a_act_vload4
: backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a;
int ai = 0;
CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q6_K->ql));
CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q6_K->qh));
@@ -23925,7 +23953,11 @@ static void ggml_cl_mul_mat_q5_K_f32_adreno(ggml_backend_t backend, const ggml_t
size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) };
backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst);
cl_kernel dk = backend_ctx->kernel_gemm_noshuffle_q5_k_q8_1_dp4a;
static const char * q5_k_act_vload4_env = getenv("GGML_OPENCL_Q5_K_DP4A_ACT_VLOAD4");
const bool q5_k_act_vload4_on = q5_k_act_vload4_env == nullptr || atoi(q5_k_act_vload4_env) != 0;
cl_kernel dk = q5_k_act_vload4_on
? backend_ctx->kernel_gemm_noshuffle_q5_k_q8_1_dp4a_act_vload4
: backend_ctx->kernel_gemm_noshuffle_q5_k_q8_1_dp4a;
int ai = 0;
CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_k->q));
CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_k->qh));
ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_q8_1_dp4a.cl
+126
View File
@@ -125,3 +125,129 @@ kernel void kernel_gemm_noshuffle_q4_0_q8_1_dp4a(
}
#undef NGROUPS
}
// Stage each 32-K activation block as two uint4 values.
inline int dot8_q8a_v(uint8 qw, uint4 a0, uint4 a1) {
int r = 0;
r = dot_acc_sat_4x8packed_ss_int(qw.s0, a0.x, r);
r = dot_acc_sat_4x8packed_ss_int(qw.s1, a0.y, r);
r = dot_acc_sat_4x8packed_ss_int(qw.s2, a0.z, r);
r = dot_acc_sat_4x8packed_ss_int(qw.s3, a0.w, r);
r = dot_acc_sat_4x8packed_ss_int(qw.s4, a1.x, r);
r = dot_acc_sat_4x8packed_ss_int(qw.s5, a1.y, r);
r = dot_acc_sat_4x8packed_ss_int(qw.s6, a1.z, r);
r = dot_acc_sat_4x8packed_ss_int(qw.s7, a1.w, r);
return r;
}
__attribute__((qcom_wave_pair_mode(1)))
kernel void kernel_gemm_noshuffle_q4_0_q8_1_dp4a_act_vload4(
__global const ushort * src0_q,
__global const half * src0_d,
__global const uint * src1_qa,
__global const half * src1_da,
__global const half * src1_sa,
__global float * dst,
ulong offsetd,
int m,
int n_no_padding,
int k
) {
dst = (global float *)((global char *)dst + offsetd);
const uint lid = get_local_id(0);
const uint block_id_m = get_global_id(1);
const uint block_id_n = get_global_id(2);
const uint row = block_id_m * 64 + lid;
const uint col_base = block_id_n * TILESIZE_N;
const bool row_valid = row < (uint)m;
const uint rrow = row_valid ? row : 0;
const uint k_u = (uint)k >> 2;
const uint k_b = (uint)k >> 5;
__local uint4 sh_qa4[TILESIZE_N][2];
__local half sh_d[TILESIZE_N];
__local half sh_s[TILESIZE_N];
#define NGROUPS (TILESIZE_N / 4)
float4 acc[NGROUPS];
#pragma unroll
for (int g = 0; g < NGROUPS; ++g) {
acc[g] = (float4)(0.0f);
}
for (uint step = 0; step < (uint)k; step += 32) {
const uint sub = step >> 5;
const float d_w = (float)src0_d[rrow + sub * (uint)m];
const uint qsbase = rrow + (step >> 2) * (uint)m;
uint8 qw;
qw.s0 = EXP4(src0_q[qsbase + 0 * m]);
qw.s1 = EXP4(src0_q[qsbase + 1 * m]);
qw.s2 = EXP4(src0_q[qsbase + 2 * m]);
qw.s3 = EXP4(src0_q[qsbase + 3 * m]);
qw.s4 = EXP4(src0_q[qsbase + 4 * m]);
qw.s5 = EXP4(src0_q[qsbase + 5 * m]);
qw.s6 = EXP4(src0_q[qsbase + 6 * m]);
qw.s7 = EXP4(src0_q[qsbase + 7 * m]);
for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) {
const uint t = idx >> 1;
const uint v = idx & 1;
const uint c = col_base + t;
sh_qa4[t][v] = (c < (uint)n_no_padding)
? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2))
: (uint4)(0u);
}
if (lid < TILESIZE_N) {
const uint c = col_base + lid;
sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0;
sh_s[lid] = (c < (uint)n_no_padding) ? src1_sa[c * k_b + sub] : (half)0;
}
barrier(CLK_LOCAL_MEM_FENCE);
#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3]))
#pragma unroll
for (int g = 0; g < NGROUPS; ++g) {
const int b = g * 4;
float4 rf;
#define DOTV(T) dot8_q8a_v(qw, sh_qa4[T][0], sh_qa4[T][1])
rf.s0 = (float)DOTV(b+0);
rf.s1 = (float)DOTV(b+1);
rf.s2 = (float)DOTV(b+2);
rf.s3 = (float)DOTV(b+3);
#undef DOTV
acc[g] += d_w * (LD4(sh_d, b) * rf - 8.0f * LD4(sh_s, b));
}
#undef LD4
barrier(CLK_LOCAL_MEM_FENCE);
}
if (!row_valid) {
return;
}
#pragma unroll
for (int g = 0; g < NGROUPS; ++g) {
const uint b = (uint)(g * 4);
const float4 a = acc[g];
const uint c0 = col_base + b;
if (c0 + 0 < (uint)n_no_padding) {
dst[(c0 + 0) * (uint)m + row] = a.s0;
}
if (c0 + 1 < (uint)n_no_padding) {
dst[(c0 + 1) * (uint)m + row] = a.s1;
}
if (c0 + 2 < (uint)n_no_padding) {
dst[(c0 + 2) * (uint)m + row] = a.s2;
}
if (c0 + 3 < (uint)n_no_padding) {
dst[(c0 + 3) * (uint)m + row] = a.s3;
}
}
#undef NGROUPS
}
ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_q8_1_dp4a.cl
+138
View File
@@ -279,3 +279,141 @@ kernel void kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg(
}
#undef NGROUPS
}
// Stage each 32-K activation block as two uint4 values.
inline int dot8_q8a_v(uint8 qw, uint4 a0, uint4 a1) {
int r = 0;
r = dot_acc_sat_4x8packed_ss_int(qw.s0, a0.x, r);
r = dot_acc_sat_4x8packed_ss_int(qw.s1, a0.y, r);
r = dot_acc_sat_4x8packed_ss_int(qw.s2, a0.z, r);
r = dot_acc_sat_4x8packed_ss_int(qw.s3, a0.w, r);
r = dot_acc_sat_4x8packed_ss_int(qw.s4, a1.x, r);
r = dot_acc_sat_4x8packed_ss_int(qw.s5, a1.y, r);
r = dot_acc_sat_4x8packed_ss_int(qw.s6, a1.z, r);
r = dot_acc_sat_4x8packed_ss_int(qw.s7, a1.w, r);
return r;
}
__attribute__((qcom_wave_pair_mode(1)))
kernel void kernel_gemm_noshuffle_q4_k_q8_1_dp4a_act_vload4(
__global const ushort * src0_q,
__global const uchar * src0_s,
__global const half * src0_d,
__global const half * src0_dm,
__global const uint * src1_qa,
__global const half * src1_da,
__global const half * src1_sa,
__global float * dst,
ulong offsetd,
int m,
int n_no_padding,
int k,
uchar mask_d6,
uchar mask_d4,
uchar mask_hi2
) {
dst = (global float *)((global char *)dst + offsetd);
const uint lid = get_local_id(0);
const uint block_id_m = get_global_id(1);
const uint block_id_n = get_global_id(2);
const uint row = block_id_m * 64 + lid;
const uint col_base = block_id_n * TILESIZE_N;
const bool row_valid = row < (uint)m;
const uint rrow = row_valid ? row : 0;
const uint k_u = (uint)k >> 2;
const uint k_b = (uint)k >> 5;
__local uint4 sh_qa4[TILESIZE_N][2];
__local half sh_d[TILESIZE_N];
__local half sh_s[TILESIZE_N];
#define NGROUPS (TILESIZE_N / 4)
float4 acc[NGROUPS];
#pragma unroll
for (int g = 0; g < NGROUPS; ++g) {
acc[g] = (float4)(0.0f);
}
for (uint step = 0; step < (uint)k; step += 32) {
const uint sub = step >> 5;
const uint sb_idx = step / QK_K;
const uint sub_idx = sub & 7;
const float dd = (float)src0_d [rrow + sb_idx * m];
const float dmm = (float)src0_dm[rrow + sb_idx * m];
global const uchar * sc = src0_s + sb_idx * K_SCALE_SIZE * (uint)m + rrow;
uchar sv, mn;
get_scale_min_k4(sub_idx, sc, (uint)m, &sv, &mn, mask_d6, mask_d4, mask_hi2);
const float scale = dd * (float)sv;
const float minv = dmm * (float)mn;
const uint wbase = rrow + (step >> 2) * (uint)m;
uint8 qw;
qw.s0 = EXP4(src0_q[wbase + 0 * m]);
qw.s1 = EXP4(src0_q[wbase + 1 * m]);
qw.s2 = EXP4(src0_q[wbase + 2 * m]);
qw.s3 = EXP4(src0_q[wbase + 3 * m]);
qw.s4 = EXP4(src0_q[wbase + 4 * m]);
qw.s5 = EXP4(src0_q[wbase + 5 * m]);
qw.s6 = EXP4(src0_q[wbase + 6 * m]);
qw.s7 = EXP4(src0_q[wbase + 7 * m]);
for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) {
const uint t = idx >> 1;
const uint v = idx & 1;
const uint c = col_base + t;
sh_qa4[t][v] = (c < (uint)n_no_padding)
? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2))
: (uint4)(0u);
}
if (lid < TILESIZE_N) {
const uint c = col_base + lid;
sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0;
sh_s[lid] = (c < (uint)n_no_padding) ? src1_sa[c * k_b + sub] : (half)0;
}
barrier(CLK_LOCAL_MEM_FENCE);
#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3]))
#pragma unroll
for (int g = 0; g < NGROUPS; ++g) {
const int b = g * 4;
float4 rf;
#define DOTV(T) dot8_q8a_v(qw, sh_qa4[T][0], sh_qa4[T][1])
rf.s0 = (float)DOTV(b+0);
rf.s1 = (float)DOTV(b+1);
rf.s2 = (float)DOTV(b+2);
rf.s3 = (float)DOTV(b+3);
#undef DOTV
acc[g] += scale * LD4(sh_d, b) * rf - minv * LD4(sh_s, b);
}
#undef LD4
barrier(CLK_LOCAL_MEM_FENCE);
}
if (!row_valid) {
return;
}
#pragma unroll
for (int g = 0; g < NGROUPS; ++g) {
const uint b = (uint)(g * 4);
const float4 a = acc[g];
const uint c0 = col_base + b;
if (c0 + 0 < (uint)n_no_padding) {
dst[(c0 + 0) * (uint)m + row] = a.s0;
}
if (c0 + 1 < (uint)n_no_padding) {
dst[(c0 + 1) * (uint)m + row] = a.s1;
}
if (c0 + 2 < (uint)n_no_padding) {
dst[(c0 + 2) * (uint)m + row] = a.s2;
}
if (c0 + 3 < (uint)n_no_padding) {
dst[(c0 + 3) * (uint)m + row] = a.s3;
}
}
#undef NGROUPS
}
ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_q8_1_dp4a.cl
+138
View File
@@ -162,3 +162,141 @@ kernel void kernel_gemm_noshuffle_q5_k_q8_1_dp4a(
}
#undef NGROUPS
}
// Stage each 32-K activation block as two uint4 values.
inline int dot8_q8a_v(uint8 qw, uint4 a0, uint4 a1) {
int r = 0;
r = dot_acc_sat_4x8packed_ss_int(qw.s0, a0.x, r);
r = dot_acc_sat_4x8packed_ss_int(qw.s1, a0.y, r);
r = dot_acc_sat_4x8packed_ss_int(qw.s2, a0.z, r);
r = dot_acc_sat_4x8packed_ss_int(qw.s3, a0.w, r);
r = dot_acc_sat_4x8packed_ss_int(qw.s4, a1.x, r);
r = dot_acc_sat_4x8packed_ss_int(qw.s5, a1.y, r);
r = dot_acc_sat_4x8packed_ss_int(qw.s6, a1.z, r);
r = dot_acc_sat_4x8packed_ss_int(qw.s7, a1.w, r);
return r;
}
__attribute__((qcom_wave_pair_mode(1)))
kernel void kernel_gemm_noshuffle_q5_k_q8_1_dp4a_act_vload4(
__global const ushort * src0_q,
__global const uchar * src0_qh,
__global const uchar * src0_s,
__global const half * src0_d,
__global const half * src0_dm,
__global const uint * src1_qa,
__global const half * src1_da,
__global const half * src1_sa,
__global float * dst,
ulong offsetd,
int m,
int n_no_padding,
int k,
uchar mask_d6,
uchar mask_d4,
uchar mask_hi2
) {
dst = (global float *)((global char *)dst + offsetd);
const uint lid = get_local_id(0);
const uint block_id_m = get_global_id(1);
const uint block_id_n = get_global_id(2);
const uint row = block_id_m * 64 + lid;
const uint col_base = block_id_n * TILESIZE_N;
const bool row_valid = row < (uint)m;
const uint rrow = row_valid ? row : 0;
const uint num_superblocks = (uint)k / QK_K;
const uint k_u = (uint)k >> 2;
const uint k_b = (uint)k >> 5;
__local uint4 sh_qa4[TILESIZE_N][2];
__local half sh_d[TILESIZE_N];
__local half sh_s[TILESIZE_N];
#define NGROUPS (TILESIZE_N / 4)
float4 acc[NGROUPS];
#pragma unroll
for (int g = 0; g < NGROUPS; ++g) {
acc[g] = (float4)(0.0f);
}
for (uint step = 0; step < (uint)k; step += 32) {
const uint sub = step >> 5;
const uint sb_idx = step / QK_K;
const uint sub_idx = sub & 7;
const float dd = (float)src0_d [rrow + sb_idx * m];
const float dmm = (float)src0_dm[rrow + sb_idx * m];
global const uchar * sc = src0_s + rrow * num_superblocks * K_SCALE_SIZE + sb_idx * K_SCALE_SIZE;
uchar sv, mn;
get_scale_min_k4(sub_idx, sc, &sv, &mn, mask_d6, mask_d4, mask_hi2);
const float scale = dd * (float)sv;
const float minv = dmm * (float)mn;
const uint wbase = rrow + (step >> 2) * (uint)m;
const uint qhbase = rrow + (step >> 3) * (uint)m;
uint8 qw;
#define QWU(u) ( EXP4((uint)src0_q[wbase + (uint)(u) * m]) \
| EXP1( (uint)((src0_qh[qhbase + (uint)((u) >> 1) * m] >> (((u) & 1) * 4)) & 0x0Fu) ) )
qw.s0 = QWU(0); qw.s1 = QWU(1); qw.s2 = QWU(2); qw.s3 = QWU(3);
qw.s4 = QWU(4); qw.s5 = QWU(5); qw.s6 = QWU(6); qw.s7 = QWU(7);
#undef QWU
for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) {
const uint t = idx >> 1;
const uint v = idx & 1;
const uint c = col_base + t;
sh_qa4[t][v] = (c < (uint)n_no_padding)
? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2))
: (uint4)(0u);
}
if (lid < TILESIZE_N) {
const uint c = col_base + lid;
sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0;
sh_s[lid] = (c < (uint)n_no_padding) ? src1_sa[c * k_b + sub] : (half)0;
}
barrier(CLK_LOCAL_MEM_FENCE);
#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3]))
#pragma unroll
for (int g = 0; g < NGROUPS; ++g) {
const int b = g * 4;
float4 rf;
#define DOTV(T) dot8_q8a_v(qw, sh_qa4[T][0], sh_qa4[T][1])
rf.s0 = (float)DOTV(b+0);
rf.s1 = (float)DOTV(b+1);
rf.s2 = (float)DOTV(b+2);
rf.s3 = (float)DOTV(b+3);
#undef DOTV
acc[g] += scale * LD4(sh_d, b) * rf - minv * LD4(sh_s, b);
}
#undef LD4
barrier(CLK_LOCAL_MEM_FENCE);
}
if (!row_valid) {
return;
}
#pragma unroll
for (int g = 0; g < NGROUPS; ++g) {
const uint b = (uint)(g * 4);
const float4 a = acc[g];
const uint c0 = col_base + b;
if (c0 + 0 < (uint)n_no_padding) {
dst[(c0 + 0) * (uint)m + row] = a.s0;
}
if (c0 + 1 < (uint)n_no_padding) {
dst[(c0 + 1) * (uint)m + row] = a.s1;
}
if (c0 + 2 < (uint)n_no_padding) {
dst[(c0 + 2) * (uint)m + row] = a.s2;
}
if (c0 + 3 < (uint)n_no_padding) {
dst[(c0 + 3) * (uint)m + row] = a.s3;
}
}
#undef NGROUPS
}
ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_q8_1_dp4a.cl
+114
View File
@@ -142,3 +142,117 @@ kernel void kernel_gemm_noshuffle_q6_k_q8_1_dp4a(
}
#undef NGROUPS
}
// Stage each 32-K activation block as two uint4 values.
__attribute__((qcom_wave_pair_mode(1)))
kernel void kernel_gemm_noshuffle_q6_k_q8_1_dp4a_act_vload4(
__global const ushort * src0_ql,
__global const uchar * src0_qh,
__global const ushort * src0_s,
__global const half * src0_d,
__global const uint * src1_qa,
__global const half * src1_da,
__global float * dst,
ulong offsetd,
int m,
int n_no_padding,
int k
) {
dst = (global float *)((global char *)dst + offsetd);
const uint lid = get_local_id(0);
const uint block_id_m = get_global_id(1);
const uint block_id_n = get_global_id(2);
const uint row = block_id_m * 64 + lid;
const uint col_base = block_id_n * TILESIZE_N;
const bool row_valid = row < (uint)m;
const uint rrow = row_valid ? row : 0;
const uint k_u = (uint)k >> 2;
const uint k_b = (uint)k >> 5;
__local uint4 sh_qa4[TILESIZE_N][2];
__local half sh_d[TILESIZE_N];
#define NGROUPS (TILESIZE_N / 4)
float4 acc[NGROUPS];
#pragma unroll
for (int g = 0; g < NGROUPS; ++g) {
acc[g] = (float4)(0.0f);
}
for (uint step = 0; step < (uint)k; step += 32) {
const uint sub = step >> 5;
const uint sb_idx = step / QK_K;
const float dd = (float)src0_d[rrow + sb_idx * m];
const char2 sc = as_char2(src0_s[rrow + sub * m]);
const float scale0 = dd * (float)sc.s0;
const float scale1 = dd * (float)sc.s1;
const uint wbase = rrow + (step >> 2) * (uint)m;
uint qw[8];
#pragma unroll
for (int u = 0; u < 8; ++u) {
const uint o = wbase + (uint)u * (uint)m;
qw[u] = SIGN6(EXP4((uint)src0_ql[o]) | EXP2((uint)src0_qh[o]));
}
for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) {
const uint t = idx >> 1;
const uint v = idx & 1;
const uint c = col_base + t;
sh_qa4[t][v] = (c < (uint)n_no_padding)
? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2))
: (uint4)(0u);
}
if (lid < TILESIZE_N) {
const uint c = col_base + lid;
sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0;
}
barrier(CLK_LOCAL_MEM_FENCE);
#pragma unroll
for (int g = 0; g < NGROUPS; ++g) {
const int b = g * 4;
float4 rf;
#define DOT_TOK(j) { \
const uint4 a0 = sh_qa4[b + (j)][0]; \
const uint4 a1 = sh_qa4[b + (j)][1]; \
const int raw1 = dot4_q8a(qw[0], qw[1], qw[2], qw[3], a0.x, a0.y, a0.z, a0.w); \
const int raw2 = dot4_q8a(qw[4], qw[5], qw[6], qw[7], a1.x, a1.y, a1.z, a1.w); \
rf.s##j = scale0 * (float)raw1 + scale1 * (float)raw2; \
}
DOT_TOK(0); DOT_TOK(1); DOT_TOK(2); DOT_TOK(3);
#undef DOT_TOK
const float4 ad = (float4)((float)sh_d[b+0], (float)sh_d[b+1], (float)sh_d[b+2], (float)sh_d[b+3]);
acc[g] += ad * rf;
}
barrier(CLK_LOCAL_MEM_FENCE);
}
if (!row_valid) {
return;
}
#pragma unroll
for (int g = 0; g < NGROUPS; ++g) {
const uint b = (uint)(g * 4);
const float4 a = acc[g];
const uint c0 = col_base + b;
if (c0 + 0 < (uint)n_no_padding) {
dst[(c0 + 0) * (uint)m + row] = a.s0;
}
if (c0 + 1 < (uint)n_no_padding) {
dst[(c0 + 1) * (uint)m + row] = a.s1;
}
if (c0 + 2 < (uint)n_no_padding) {
dst[(c0 + 2) * (uint)m + row] = a.s2;
}
if (c0 + 3 < (uint)n_no_padding) {
dst[(c0 + 3) * (uint)m + row] = a.s3;
}
}
#undef NGROUPS
}
ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_q8_1_dp4a.cl
+122
View File
@@ -210,3 +210,125 @@ kernel void kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg(
}
#undef NGROUPS
}
// Stage each 32-K activation block as two uint4 values.
inline int dot8_q8a_v(uint8 qw, uint4 a0, uint4 a1) {
int r = 0;
r = dot_acc_sat_4x8packed_ss_int(qw.s0, a0.x, r);
r = dot_acc_sat_4x8packed_ss_int(qw.s1, a0.y, r);
r = dot_acc_sat_4x8packed_ss_int(qw.s2, a0.z, r);
r = dot_acc_sat_4x8packed_ss_int(qw.s3, a0.w, r);
r = dot_acc_sat_4x8packed_ss_int(qw.s4, a1.x, r);
r = dot_acc_sat_4x8packed_ss_int(qw.s5, a1.y, r);
r = dot_acc_sat_4x8packed_ss_int(qw.s6, a1.z, r);
r = dot_acc_sat_4x8packed_ss_int(qw.s7, a1.w, r);
return r;
}
__attribute__((qcom_wave_pair_mode(1)))
kernel void kernel_gemm_noshuffle_q8_0_q8_1_dp4a_act_vload4(
__global const uint * src0_q,
__global const half * src0_d,
__global const uint * src1_qa,
__global const half * src1_da,
__global float * dst,
ulong offsetd,
int m,
int n_no_padding,
int k
) {
dst = (global float *)((global char *)dst + offsetd);
const uint lid = get_local_id(0);
const uint block_id_m = get_global_id(1);
const uint block_id_n = get_global_id(2);
const uint row = block_id_m * 64 + lid;
const uint col_base = block_id_n * TILESIZE_N;
const bool row_valid = row < (uint)m;
const uint rrow = row_valid ? row : 0;
const uint k_u = (uint)k >> 2;
const uint k_b = (uint)k >> 5;
__local uint4 sh_qa4[TILESIZE_N][2];
__local half sh_d[TILESIZE_N];
#define NGROUPS (TILESIZE_N / 4)
float4 acc[NGROUPS];
#pragma unroll
for (int g = 0; g < NGROUPS; ++g) {
acc[g] = (float4)(0.0f);
}
for (uint step = 0; step < (uint)k; step += 32) {
const uint sub = step >> 5;
const float d_w = (float)src0_d[rrow + sub * (uint)m];
const uint wbase = rrow + (step >> 2) * (uint)m;
uint8 qw;
qw.s0 = src0_q[wbase + 0 * m];
qw.s1 = src0_q[wbase + 1 * m];
qw.s2 = src0_q[wbase + 2 * m];
qw.s3 = src0_q[wbase + 3 * m];
qw.s4 = src0_q[wbase + 4 * m];
qw.s5 = src0_q[wbase + 5 * m];
qw.s6 = src0_q[wbase + 6 * m];
qw.s7 = src0_q[wbase + 7 * m];
for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) {
const uint t = idx >> 1;
const uint v = idx & 1;
const uint c = col_base + t;
sh_qa4[t][v] = (c < (uint)n_no_padding)
? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2))
: (uint4)(0u);
}
if (lid < TILESIZE_N) {
const uint c = col_base + lid;
sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0;
}
barrier(CLK_LOCAL_MEM_FENCE);
#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3]))
#pragma unroll
for (int g = 0; g < NGROUPS; ++g) {
const int b = g * 4;
float4 rf;
#define DOTV(T) dot8_q8a_v(qw, sh_qa4[T][0], sh_qa4[T][1])
rf.s0 = (float)DOTV(b+0);
rf.s1 = (float)DOTV(b+1);
rf.s2 = (float)DOTV(b+2);
rf.s3 = (float)DOTV(b+3);
#undef DOTV
acc[g] += d_w * LD4(sh_d, b) * rf;
}
#undef LD4
barrier(CLK_LOCAL_MEM_FENCE);
}
if (!row_valid) {
return;
}
#pragma unroll
for (int g = 0; g < NGROUPS; ++g) {
const uint b = (uint)(g * 4);
const float4 a = acc[g];
const uint c0 = col_base + b;
if (c0 + 0 < (uint)n_no_padding) {
dst[(c0 + 0) * (uint)m + row] = a.s0;
}
if (c0 + 1 < (uint)n_no_padding) {
dst[(c0 + 1) * (uint)m + row] = a.s1;
}
if (c0 + 2 < (uint)n_no_padding) {
dst[(c0 + 2) * (uint)m + row] = a.s2;
}
if (c0 + 3 < (uint)n_no_padding) {
dst[(c0 + 3) * (uint)m + row] = a.s3;
}
}
#undef NGROUPS
}