--- a/ggml/src/ggml-cuda/common.cuh +++ b/ggml/src/ggml-cuda/common.cuh @@ -293,7 +293,8 @@ #define AMPERE_MMA_AVAILABLE #endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE -#if !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_BLACKWELL && __CUDA_ARCH__ < GGML_CUDA_CC_RUBIN +// FreeToken: the native fp4 MMA needs an sm_120a/sm_120f build, so it is off unless GGML_CUDA_FP4_MMA is defined; FP4 weights then take Q8_1 activations +#if !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_BLACKWELL && __CUDA_ARCH__ < GGML_CUDA_CC_RUBIN && defined(GGML_CUDA_FP4_MMA) # define BLACKWELL_MMA_AVAILABLE #endif // !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_BLACKWELL @@ -374,8 +375,13 @@ } static bool blackwell_mma_available(const int cc) { +#ifdef GGML_CUDA_FP4_MMA return GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_BLACKWELL && ggml_cuda_highest_compiled_arch(cc) < GGML_CUDA_CC_RUBIN; +#else + GGML_UNUSED(cc); + return false; +#endif // GGML_CUDA_FP4_MMA } // Checks whether the tensor's base data pointer and higher-dimensional strides are byte-aligned to `alignment` bytes. @@ -1582,6 +1588,9 @@ const void * gate_bias = nullptr; const void * x_scale = nullptr; const void * gate_scale = nullptr; + // FreeToken: element strides of x_scale / gate_scale, so they can be columns of a per-expert scale bank + uint32_t x_scale_stride = 1; + uint32_t gate_scale_stride = 1; ggml_glu_op glu_op; float glu_limit = 0.0f; const void * shared_up = nullptr; --- a/ggml/src/ggml-cuda/dequantize.cuh +++ b/ggml/src/ggml-cuda/dequantize.cuh @@ -450,3 +450,15 @@ y[j+16] = ggml_cuda_cast(d * kvalues_mxfp4[q4[j] >> 4]*0.5f); } } + +// FreeToken: for k_get_rows with qk = QK_NVFP4_SUB, so ib indexes a 16-element sub-block and the nibble pair lands at iqs and iqs + 8 +static __device__ __forceinline__ void dequantize_nvfp4(const void * vx, const int64_t ib, const int iqs, float2 & v){ + const block_nvfp4 * x = (const block_nvfp4 *) vx + ib/(QK_NVFP4/QK_NVFP4_SUB); + const int sub = ib % (QK_NVFP4/QK_NVFP4_SUB); + + const float d = ggml_cuda_ue4m3_to_fp32(x->d[sub]); + const int vui = x->qs[sub*(QK_NVFP4_SUB/2) + iqs]; + + v.x = d * kvalues_mxfp4[vui & 0xF]; + v.y = d * kvalues_mxfp4[vui >> 4]; +} --- a/ggml/src/ggml-cuda/getrows.cu +++ b/ggml/src/ggml-cuda/getrows.cu @@ -404,6 +404,10 @@ get_rows_cuda_kq<32, dst_t, dequantize_mxfp4>(src0_d, src1_d, dst_d, ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream); break; + case GGML_TYPE_NVFP4: + get_rows_cuda_q(src0_d, src1_d, dst_d, + ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream); + break; default: GGML_ABORT("%s: unsupported src0 type: %s\n", __func__, ggml_type_name(src0_type)); } --- a/ggml/src/ggml-cuda/mmid.cu +++ b/ggml/src/ggml-cuda/mmid.cu @@ -49,7 +49,7 @@ int iex_used = -1; // The index at which the expert is used, if any. for (int iex = threadIdx.x; iex < n_expert_used; iex += warp_size) { const int expert_used = ids[it*si1 + iex]; - nex_prev += expert_used < expert; + nex_prev += (unsigned) expert_used < (unsigned) expert; // FreeToken: unsigned, so an inactive route (negative id) sorts after every expert if (expert_used == expert) { iex_used = iex; } @@ -75,7 +75,7 @@ const int expert_used = (neu_padded == n_expert_used || iex < n_expert_used) && it < n_tokens ? ids[it*si1 + iex] : INT_MAX; const int iex_used = expert_used == expert ? iex : -1; - nex_prev += expert_used < expert; + nex_prev += (unsigned) expert_used < (unsigned) expert; // FreeToken: unsigned, so an inactive route (negative id) sorts after every expert // Whether the threads at this token position have used the expert: const int it_compact_add_self = warp_reduce_any(iex_used != -1); @@ -177,3 +177,45 @@ break; } } + +// Lists the (expert, column tile) pairs that hold columns for a flat MoE grid, in expert order. +// The other ntiles grid columns get tile jt_none of expert 0, which starts past its columns, so mul_mat_q skips them. +template +__launch_bounds__(nthreads, 1) +static __global__ void mm_ids_tile_map( + const int32_t * __restrict__ expert_bounds, int2 * __restrict__ tile_map, const int n_experts, const int J, const int ntiles, const int jt_none) { + __shared__ int scan[nthreads]; + + int start = 0; // First tile of the current chunk of experts. + for (int e0 = 0; e0 < n_experts; e0 += nthreads) { + const int e = e0 + threadIdx.x; + const int n = e < n_experts ? (expert_bounds[e + 1] - expert_bounds[e] + J - 1) / J : 0; + + scan[threadIdx.x] = n; + __syncthreads(); +#pragma unroll + for (int offset = 1; offset < nthreads; offset *= 2) { + const int add = threadIdx.x >= offset ? scan[threadIdx.x - offset] : 0; + __syncthreads(); + scan[threadIdx.x] += add; + __syncthreads(); + } + + const int first = start + scan[threadIdx.x] - n; + for (int jt = 0; jt < n; ++jt) { + tile_map[first + jt] = make_int2(e, jt); + } + start += scan[nthreads - 1]; + __syncthreads(); + } + + for (int t = start + threadIdx.x; t < ntiles; t += nthreads) { + tile_map[t] = make_int2(0, jt_none); + } +} + +void ggml_cuda_launch_mm_ids_tile_map( + const int32_t * expert_bounds, int2 * tile_map, const int n_experts, const int J, const int ntiles, const int jt_none, cudaStream_t stream) { + constexpr int nthreads = 512; + mm_ids_tile_map<<<1, nthreads, 0, stream>>>(expert_bounds, tile_map, n_experts, J, ntiles, jt_none); +} --- a/ggml/src/ggml-cuda/mmid.cuh +++ b/ggml/src/ggml-cuda/mmid.cuh @@ -3,3 +3,6 @@ void ggml_cuda_launch_mm_ids_helper( const int32_t * ids, int32_t * ids_src1, int32_t * ids_dst, int32_t * expert_bounds, int n_experts, int n_tokens, int n_expert_used, int nchannels_y, int si1, int sis1, bool write_inverse, cudaStream_t stream); + +void ggml_cuda_launch_mm_ids_tile_map( + const int32_t * expert_bounds, int2 * tile_map, int n_experts, int J, int ntiles, int jt_none, cudaStream_t stream); --- a/ggml/src/ggml-cuda/mmq-load-tiles.cuh +++ b/ggml/src/ggml-cuda/mmq-load-tiles.cuh @@ -1095,6 +1095,76 @@ } } +// FreeToken: IQ1_M's delta changes every 8 values, so it cannot use IQ1_S's per-32 dm. 8 * (q + delta) is an integer +// (8q - 7, or 8q - 9 with the shift bit), so the tile holds those int8 values and d * scale / 8 per 16, like IQ2_XS. +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq1_m( + const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2); +#else + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ1_M, I); + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + txs.qs); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + + constexpr int threads_per_row = MMQ_ITER_K / (4 * QR1_M); + constexpr int nrows = warp_size / threads_per_row; + const int kqsx = threadIdx.x % threads_per_row; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nwarps * nrows) { + int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row; + + if (fallback) { + i = min(i, i_max); + } + + const block_iq1_m * bxi = (const block_iq1_m *) x + kbx0 + i*stride; + + const int qs_packed = get_int_b4(bxi->qs, kqsx); + const uint8_t * qs = (const uint8_t *) &qs_packed; + + const int qh = bxi->qh[2*kqsx + 0] | (bxi->qh[2*kqsx + 1] << 8); + + #pragma unroll + for (int l = 0; l < QR1_M/2; ++l) { + const int qhl = qh >> (4*l); + const int grid = iq1s_grid_gpu[qs[l] | ((qhl & 0x07) << 8)]; + const int off = qhl & 0x08 ? 0x09090909 : 0x07070707; + + const int grid0 = __vsub4(((grid >> 0) & 0x0F0F0F0F) << 3, off); + const int grid1 = __vsub4(((grid >> 4) & 0x0F0F0F0F) << 3, off); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_qs[i*sram_stride + 8*kqsx + (2*l + 0)] = grid0; + x_qs[i*sram_stride + 8*kqsx + (2*l + 1)] = grid1; +#else + x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 0)] = grid0; + x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 1)] = grid1; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } + + const uint16_t * sc = (const uint16_t *) bxi->scales; + iq1m_scale_t scale; + scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00F0) | ((sc[2] >> 4) & 0x0F00) | (sc[3] & 0xF000); + const float d = __half2float(scale.f16) / 8; + const int ls = sc[kqsx/2] >> (6*(kqsx%2)); +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_df[i*sram_stride + 2*kqsx+0] = d * (2*((ls >> 0) & 0x07) + 1); + x_df[i*sram_stride + 2*kqsx+1] = d * (2*((ls >> 3) & 0x07) + 1); +#else + x_df[i*(2*MMQ_TILE_NE_K*2/QI8_0) + i/(QI8_0/4) + 2*kqsx+0] = d * (2*((ls >> 0) & 0x07) + 1); + x_df[i*(2*MMQ_TILE_NE_K*2/QI8_0) + i/(QI8_0/4) + 2*kqsx+1] = d * (2*((ls >> 3) & 0x07) + 1); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } +} + template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq2_xxs( const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); --- a/ggml/src/ggml-cuda/mmq.cu +++ b/ggml/src/ggml-cuda/mmq.cu @@ -48,6 +48,9 @@ case GGML_TYPE_IQ1_S: mul_mat_q_case(ctx, args, stream); break; + case GGML_TYPE_IQ1_M: + mul_mat_q_case(ctx, args, stream); + break; case GGML_TYPE_IQ2_XXS: mul_mat_q_case(ctx, args, stream); break; @@ -132,9 +135,11 @@ } void ggml_cuda_mul_mat_q( - ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst) { - GGML_ASSERT( src1->type == GGML_TYPE_F32); - GGML_ASSERT( dst->type == GGML_TYPE_F32); + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst, + int64_t ncols_max, bool inactive_ids) { + // FreeToken: bf16 and fp16 activations are quantized directly, except for the native fp4 paths; dst is fp32, bf16 or fp16 + GGML_ASSERT( src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_BF16 || src1->type == GGML_TYPE_F16); + GGML_ASSERT( dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_BF16 || dst->type == GGML_TYPE_F16); GGML_ASSERT(!ids || ids->type == GGML_TYPE_I32); // Optional, used for batched GGML_MUL_MAT_ID. GGML_TENSOR_BINARY_OP_LOCALS; @@ -156,7 +161,9 @@ const char * src0_d = (const char *) src0->data; const float * src1_d = (const float *) src1->data; - float * dst_d = (float *) dst->data; + const mmq_dst dst_d = {dst->data, dst->type}; + const nv_bfloat16 * src1_bf16 = src1->type == GGML_TYPE_BF16 ? (const nv_bfloat16 *) src1->data : nullptr; + const half * src1_f16 = src1->type == GGML_TYPE_F16 ? (const half *) src1->data : nullptr; // If src0 is a temporary compute buffer, clear any potential padding. if (ggml_backend_buffer_get_usage(src0->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE) { @@ -183,6 +190,7 @@ const ggml_prec prec_src1 = ggml_cuda_mmq_get_prec_src1(src0, dst, cc); const bool use_native_fp4 = prec_src1 == GGML_PREC_Q4; + GGML_ASSERT(!use_native_fp4 || src1->type == GGML_TYPE_F32); const size_t y_block_size = use_native_fp4 ? sizeof(block_fp4_mmq) : sizeof(block_q8_1_mmq); const size_t y_values_per_block = use_native_fp4 ? QK_FP4_MMQ : QK8_1_MMQ; @@ -192,7 +200,8 @@ int64_t ncols_opt = ne11; if (ids) { const int64_t n_expert_used = ids->ne[0]; - ncols_opt = ne12; + // a flat grid runs only the tiles that hold tokens, so it sizes them for the average expert + ncols_opt = ncols_max > 0 ? ncols_max : (ne12*n_expert_used + ne02 - 1) / ne02; // Each expert only sees ne12*n_expert_used/ne02 tokens on average. // On RDNA3 and RDNA4 it is faster to pick the tile size against this value instead of ne12. @@ -249,6 +258,12 @@ quantize_mmq_fp4_cuda(src1_d, nullptr, src1_q8_1.get(), src1_scale.ptr, src0->type, use_aligned_float8, ne10, s11, s12, s13, ne10_padded, ne11, ne12, ne13, stream); + } else if (src1_bf16) { + quantize_mmq_q8_1_cuda(src1_bf16, nullptr, src1_q8_1.get(), src0->type, ne10, s11, s12, s13, ne10_padded, + ne11, ne12, ne13, stream); + } else if (src1_f16) { + quantize_mmq_q8_1_cuda(src1_f16, nullptr, src1_q8_1.get(), src0->type, ne10, s11, s12, s13, ne10_padded, + ne11, ne12, ne13, stream); } else { quantize_mmq_q8_1_cuda(src1_d, nullptr, src1_q8_1.get(), src0->type, ne10, s11, s12, s13, ne10_padded, ne11, ne12, ne13, stream); @@ -284,6 +299,10 @@ ggml_cuda_pool_alloc ids_src1(ctx.pool(), ne_get_rows); ggml_cuda_pool_alloc ids_dst(ctx.pool(), ne_get_rows + J_best-1); // Needs to be padded for unconditional memory access. ggml_cuda_pool_alloc expert_bounds(ctx.pool(), ne02 + 1); + // FreeToken: inactive routes (negative ids, only with inactive_ids) get no compact row, so their ids_src1 entries stay -1 + if (inactive_ids) { + CUDA_CHECK(cudaMemsetAsync(ids_src1.get(), 0xFF, ne_get_rows*sizeof(int32_t), stream)); + } // gate/up activations are broadcast across experts (ne11 == 1): quantize each token once and // scatter to its slots. ids_src1 then holds the inverse map (token slot -> compact row). @@ -299,6 +318,17 @@ CUDA_CHECK(cudaGetLastError()); } + // Every expert's column tiles fit in ntx_flat grid columns, so a flat grid needs no host sync to size it. + ggml_cuda_pool_alloc tile_map(ctx.pool()); + int ntx_flat = 0; + if (ncols_max <= 0) { + const int jt_none = (ne_get_rows + J_best - 1) / J_best; + ntx_flat = jt_none + std::min(ne02, ne_get_rows); + tile_map.alloc(ntx_flat); + ggml_cuda_launch_mm_ids_tile_map(expert_bounds.get(), tile_map.get(), ne02, J_best, ntx_flat, jt_none, stream); + CUDA_CHECK(cudaGetLastError()); + } + const size_t nbytes_src1_q8_1 = ne12*n_expert_used*ne10_padded * y_block_size/y_values_per_block + src1_q8_1_padding; ggml_cuda_pool_alloc src1_q8_1(ctx.pool(), nbytes_src1_q8_1); ggml_cuda_pool_alloc src1_scale(ctx.pool()); @@ -325,9 +355,21 @@ quantize_mmq_fp4_cuda(src1_d, ids_src1.get(), src1_q8_1.get(), src1_scale.ptr, src0->type, use_aligned_float8, ne10, s11, s12, s13, ne10_padded, ne11_flat, ne12_flat, ne13_flat, stream); } + } else if (dedup_bcast && src1_bf16) { + quantize_scatter_mmq_q8_1_cuda(src1_bf16, ids_src1.get(), src1_q8_1.get(), src0->type, ne10, + /*stride_token=*/s12, ne10_padded, ne12, ne11_flat, n_expert_used, stream); + } else if (dedup_bcast && src1_f16) { + quantize_scatter_mmq_q8_1_cuda(src1_f16, ids_src1.get(), src1_q8_1.get(), src0->type, ne10, + /*stride_token=*/s12, ne10_padded, ne12, ne11_flat, n_expert_used, stream); } else if (dedup_bcast) { quantize_scatter_mmq_q8_1_cuda(src1_d, ids_src1.get(), src1_q8_1.get(), src0->type, ne10, /*stride_token=*/s12, ne10_padded, ne12, ne11_flat, n_expert_used, stream); + } else if (src1_bf16) { + quantize_mmq_q8_1_cuda(src1_bf16, ids_src1.get(), src1_q8_1.get(), src0->type, ne10, s11, s12, s13, + ne10_padded, ne11_flat, ne12_flat, ne13_flat, stream); + } else if (src1_f16) { + quantize_mmq_q8_1_cuda(src1_f16, ids_src1.get(), src1_q8_1.get(), src0->type, ne10, s11, s12, s13, + ne10_padded, ne11_flat, ne12_flat, ne13_flat, stream); } else { quantize_mmq_q8_1_cuda(src1_d, ids_src1.get(), src1_q8_1.get(), src0->type, ne10, s11, s12, s13, ne10_padded, ne11_flat, ne12_flat, ne13_flat, stream); @@ -347,7 +389,7 @@ ne00, ne01, ne_get_rows, s01, ne_get_rows, s1, ne02, ne02, s02, s12, s2, ne03, ne13, s03, s13, s3, - ne12, J_best}; + ncols_max > 0 ? ncols_max : ne12, J_best, tile_map.ptr, ntx_flat}; ggml_cuda_mul_mat_q_switch_type(ctx, args, stream, prec_src1); } @@ -375,6 +417,7 @@ case GGML_TYPE_Q6_K: // ------------------------------------------------- case GGML_TYPE_IQ1_S: + case GGML_TYPE_IQ1_M: case GGML_TYPE_IQ2_XXS: case GGML_TYPE_IQ2_XS: case GGML_TYPE_IQ2_S: --- a/ggml/src/ggml-cuda/mmq.cuh +++ b/ggml/src/ggml-cuda/mmq.cuh @@ -10,6 +10,17 @@ #define MMQ_ITER_K_FP4 512 #define MMQ_NWARPS 8 +// FreeToken: MMQ writes fp32, bf16 or fp16 results (dst_t); the sums stay fp32 until the store +template static __device__ __forceinline__ T mmq_cast(float v); +template <> __device__ __forceinline__ float mmq_cast(float v) { return v; } +template <> __device__ __forceinline__ nv_bfloat16 mmq_cast(float v) { return __float2bfloat16(v); } +template <> __device__ __forceinline__ half mmq_cast(float v) { return __float2half(v); } + +struct mmq_dst { + void * p; + ggml_type type; +}; + typedef void (*ggml_cuda_mmq_load_tiles_t)(const char * __restrict__ x, int * x_tile, const int kbx0, const int i_max, const int stride); typedef void (*ggml_cuda_mmq_vec_dot_t)(const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00); typedef void (*ggml_cuda_mmq_write_back_t)(const float * __restrict__ sum, const int32_t * __restrict__ get_rows_to_sorted, @@ -88,6 +99,7 @@ case GGML_TYPE_IQ2_S: case GGML_TYPE_IQ3_XXS: case GGML_TYPE_IQ3_S: + case GGML_TYPE_IQ1_M: return MMQ_Q8_1_DS_LAYOUT_D4; case GGML_TYPE_IQ1_S: return MMQ_Q8_1_DS_LAYOUT_DS4; @@ -227,6 +239,14 @@ #undef CASE static __host__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(const ggml_type type, const int J, const bool fallback, const int cc, const ggml_prec prec_src1) { + // FreeToken: IQ1_M fills the same tile as IQ2_XS (int8 values, one scale per 16), so it takes IQ2_XS's configs + if (type == GGML_TYPE_IQ1_M) { + ggml_cuda_mmq_config config = ggml_cuda_mmq_get_config(GGML_TYPE_IQ2_XS, J, fallback, cc, prec_src1); + if (config.type == GGML_TYPE_IQ2_XS) { + config.type = GGML_TYPE_IQ1_M; + } + return config; + } if (GGML_CUDA_CC_IS_AMD(cc)) { if (GGML_CUDA_CC_IS_GCN(cc)) { return ggml_cuda_mmq_get_config_gcn(type, J, fallback); @@ -262,6 +282,13 @@ } static constexpr __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(ggml_type type, int J, bool fallback, ggml_prec prec_src1) { + if (type == GGML_TYPE_IQ1_M) { + ggml_cuda_mmq_config config = ggml_cuda_mmq_get_config(GGML_TYPE_IQ2_XS, J, fallback, prec_src1); + if (config.type == GGML_TYPE_IQ2_XS) { + config.type = GGML_TYPE_IQ1_M; + } + return config; + } #ifdef GGML_USE_HIP #ifdef GCN return ggml_cuda_mmq_get_config_gcn(type, J, fallback); @@ -409,6 +436,7 @@ case GGML_TYPE_IQ3_XXS: return MMQ_DP4A_TXS_Q8_0; case GGML_TYPE_IQ3_S: return MMQ_DP4A_TXS_Q8_0; case GGML_TYPE_IQ1_S: return MMQ_DP4A_TXS_Q8_0; + case GGML_TYPE_IQ1_M: return MMQ_DP4A_TXS_Q8_0_16; case GGML_TYPE_IQ4_XS: return MMQ_DP4A_TXS_Q8_0; case GGML_TYPE_IQ4_NL: return MMQ_DP4A_TXS_Q8_0; default: return tile_x_sizes{0, 0, 0}; @@ -429,8 +457,8 @@ #include "mmq-load-tiles.cuh" #include "mmq-vec-dot.cuh" -template static __device__ __forceinline__ void ggml_cuda_mmq_write_back_dp4a( - const float * __restrict__ sum, const int32_t * __restrict__ ids_dst, float * __restrict__ dst, +template static __device__ __forceinline__ void ggml_cuda_mmq_write_back_dp4a( + const float * __restrict__ sum, const int32_t * __restrict__ ids_dst, dst_t * __restrict__ dst, const float * __restrict__ y_scale, const int stride, const int i_max, const int j_max) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1) / warp_size; @@ -456,21 +484,21 @@ if constexpr (type == GGML_TYPE_NVFP4) { if (y_scale_used) { - dst[ids_dst[j]*stride + i] = y_scale[j] * sum[(j0/nwarps) * (I/warp_size) + i0/warp_size]; + dst[ids_dst[j]*stride + i] = mmq_cast(y_scale[j] * sum[(j0/nwarps) * (I/warp_size) + i0/warp_size]); } else { - dst[ids_dst[j]*stride + i] = sum[(j0/nwarps) * (I/warp_size) + i0/warp_size]; + dst[ids_dst[j]*stride + i] = mmq_cast(sum[(j0/nwarps) * (I/warp_size) + i0/warp_size]); } } else { - dst[ids_dst[j]*stride + i] = sum[(j0/nwarps) * (I/warp_size) + i0/warp_size]; + dst[ids_dst[j]*stride + i] = mmq_cast(sum[(j0/nwarps) * (I/warp_size) + i0/warp_size]); GGML_UNUSED(y_scale_used); } } } } -template +template static __device__ __forceinline__ void ggml_cuda_mmq_write_back_mma( - const float * __restrict__ sum, const int * __restrict__ ids_dst, float * __restrict__ dst, + const float * __restrict__ sum, const int * __restrict__ ids_dst, dst_t * __restrict__ dst, const float * __restrict__ y_scale, const int stride, const int i_max, const int j_max) { #if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) @@ -506,12 +534,12 @@ if constexpr (type == GGML_TYPE_NVFP4) { if (y_scale_used) { - dst[ids_dst[j]*stride + i] = y_scale[j] * sum[(j0/tile_C::J + n)*tile_C::ne + l]; + dst[ids_dst[j]*stride + i] = mmq_cast(y_scale[j] * sum[(j0/tile_C::J + n)*tile_C::ne + l]); } else { - dst[ids_dst[j]*stride + i] = sum[(j0/tile_C::J + n)*tile_C::ne + l]; + dst[ids_dst[j]*stride + i] = mmq_cast(sum[(j0/tile_C::J + n)*tile_C::ne + l]); } } else { - dst[ids_dst[j]*stride + i] = sum[(j0/tile_C::J + n)*tile_C::ne + l]; + dst[ids_dst[j]*stride + i] = mmq_cast(sum[(j0/tile_C::J + n)*tile_C::ne + l]); GGML_UNUSED(y_scale_used); } } @@ -617,6 +645,12 @@ ggml_cuda_mmq_load_tiles_iq1_s, ggml_cuda_mmq_vec_dot_q8_1_q8_1_dp4a, ggml_cuda_mmq_write_back_dp4a); + case GGML_TYPE_IQ1_M: + return ggml_cuda_mmq_util_funcs( + VDR_IQ2_XS_Q8_1_MMQ, + ggml_cuda_mmq_load_tiles_iq1_m, + ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_dp4a, + ggml_cuda_mmq_write_back_dp4a); case GGML_TYPE_IQ2_XXS: return ggml_cuda_mmq_util_funcs( VDR_IQ2_XXS_Q8_1_MMQ, @@ -787,6 +821,12 @@ ggml_cuda_mmq_load_tiles_iq1_s, ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma, ggml_cuda_mmq_write_back_mma); + case GGML_TYPE_IQ1_M: + return ggml_cuda_mmq_util_funcs( + -1, + ggml_cuda_mmq_load_tiles_iq1_m, + ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma, + ggml_cuda_mmq_write_back_mma); case GGML_TYPE_IQ2_XXS: return ggml_cuda_mmq_util_funcs( -1, @@ -869,10 +909,10 @@ // --------------------------------------------------------------------------------------------- -template +template static __device__ __forceinline__ void mul_mat_q_process_tile( const char * __restrict__ x, const int offset_x, const int * __restrict__ y, - const int * __restrict__ ids_dst, float * __restrict__ dst, float * __restrict__ tmp_fixup, + const int * __restrict__ ids_dst, dst_t * __restrict__ dst, float * __restrict__ tmp_fixup, const float * __restrict__ y_scale, const int stride_row_x, const int ncols_y, const int stride_col_dst, const int tile_x_max_i, const int tile_y_max_j, const int kb0_start, const int kb0_stop) { @@ -942,19 +982,32 @@ if (fixup) { write_back(sum, ids_dst, tmp_fixup + blockIdx.x*(J*I), y_scale, I, I, J); - } else { + } else if constexpr (std::is_same_v) { write_back(sum, ids_dst, dst, y_scale, stride_col_dst, tile_x_max_i, tile_y_max_j); + } else if constexpr (ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).use_mma_data_layout()) { + ggml_cuda_mmq_write_back_mma(sum, ids_dst, dst, y_scale, stride_col_dst, tile_x_max_i, tile_y_max_j); + } else { + ggml_cuda_mmq_write_back_dp4a(sum, ids_dst, dst, y_scale, stride_col_dst, tile_x_max_i, tile_y_max_j); } } +// A flat MoE grid runs only the (expert, column tile) pairs listed in tile_map, see ggml_cuda_launch_mm_ids_tile_map. +static __device__ __forceinline__ void mmq_flat_tile(const int2 * __restrict__ tile_map, int & zt, int & jt) { + if (tile_map) { + const int2 t = tile_map[jt]; + zt = t.x; + jt = t.y; + } +} + // The mul_mat_q kernel implements "stream-k" work partitioning as described in https://arxiv.org/abs/2301.03598 -template +template __launch_bounds__(ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1), ggml_cuda_mmq_get_occupancy(type, J, fallback, prec_src1)) static __global__ void mul_mat_q( const char * __restrict__ x, const int * __restrict__ y, const int32_t * __restrict__ ids_dst, - const int32_t * __restrict__ expert_bounds, float * __restrict__ dst, float * __restrict__ tmp_fixup, + const int32_t * __restrict__ expert_bounds, const int2 * __restrict__ tile_map, dst_t * __restrict__ dst, float * __restrict__ tmp_fixup, const float * __restrict__ y_scale, const uint3 blocks_per_ne00, const int nrows_x, const int ncols_dst, const int stride_row_x, const int ncols_y, const int stride_col_dst, const uint3 channel_ratio, const uint3 nchannels_y, const int stride_channel_x, const int stride_channel_y, const int stride_channel_dst, @@ -993,9 +1046,10 @@ if constexpr (!ggml_cuda_mmq_get_stream_k(type, J, fallback, prec_src1)) { const uint2 tmp2 = fast_div_modulo(blockIdx.z, nchannels_y); const int wt = tmp2.x; - const int zt = tmp2.y; - const int jt = blockIdx.y; + int zt = tmp2.y; + int jt = blockIdx.y; const int it = blockIdx.x; + mmq_flat_tile(tile_map, zt, jt); // Defaults for regular matrix multiplication: int col_low = 0; @@ -1076,14 +1130,15 @@ while (kbc < kbc_stop && kb0_stop == int(blocks_per_ne00.z)) { int tmp = fastdiv(kbc, blocks_per_ne00); uint2 tmp2 = fast_div_modulo(tmp, ntx); - const int jt = tmp2.y; + int jt = tmp2.y; tmp = tmp2.x; tmp2 = fast_div_modulo(tmp, nchannels_y); - const int zt = tmp2.y; + int zt = tmp2.y; tmp = tmp2.x; tmp2 = fast_div_modulo(tmp, nsamples_y); const int wt = tmp2.y; const int it = tmp2.x; + mmq_flat_tile(tile_map, zt, jt); // Defaults for regular matrix multiplication: int col_low = 0; @@ -1165,14 +1220,15 @@ int tmp = fastdiv(kbc, blocks_per_ne00); uint2 tmp2 = fast_div_modulo(tmp, ntx); - const int jt = tmp2.y; + int jt = tmp2.y; tmp = tmp2.x; tmp2 = fast_div_modulo(tmp, nchannels_y); - const int zt = tmp2.y; + int zt = tmp2.y; tmp = tmp2.x; tmp2 = fast_div_modulo(tmp, nsamples_y); const int wt = tmp2.y; const int it = tmp2.x; + mmq_flat_tile(tile_map, zt, jt); // Defaults for regular matrix multiplication: int col_low = 0; @@ -1375,14 +1431,26 @@ } } +// FreeToken: copies an fp32 result with dst's layout into dst +template +static __global__ void mmq_store_dst(const float * __restrict__ src, dst_t * __restrict__ dst, const int nrows, const int stride) { + const int64_t col = blockIdx.x; + const int row = blockIdx.y*blockDim.x + threadIdx.x; + if (row < nrows) { + dst[col*stride + row] = mmq_cast(src[col*stride + row]); + } +} + struct mmq_args { - const char * x; ggml_type type_x; const int * y; const int32_t * ids_dst; const int32_t * expert_bounds; float * dst; + const char * x; ggml_type type_x; const int * y; const int32_t * ids_dst; const int32_t * expert_bounds; mmq_dst dst; const float * y_scale; int64_t ncols_x; int64_t nrows_x; int64_t ncols_dst; int64_t stride_row_x; int64_t ncols_y; int64_t nrows_dst; int64_t nchannels_x; int64_t nchannels_y; int64_t stride_channel_x; int64_t stride_channel_y; int64_t stride_channel_dst; int64_t nsamples_x; int64_t nsamples_y; int64_t stride_sample_x; int64_t stride_sample_y; int64_t stride_sample_dst; int64_t ncols_max; int J_best; // Tile width in ne11(dense)/ne12(MoE) direction to use for optimal performance. + const int2 * tile_map = nullptr; // Flat MoE grid: the (expert, column tile) of each of the ntx_flat grid columns. + int ntx_flat = 0; }; static size_t mmq_get_nbytes_shared(const ggml_cuda_mmq_config & config, const int cc) { @@ -1406,12 +1474,11 @@ const dim3 block_dims(warp_size, nwarps, 1); - CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q), nbytes_shared); - CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q), nbytes_shared); - + // a flat MoE grid puts all experts into one channel of ntx_flat column tiles + const int64_t nchannels_grid = args.tile_map ? 1 : args.nchannels_y; const int nty = (args.nrows_x + config.I - 1) / config.I; - const int ntx = (args.ncols_max + config.J - 1) / config.J; - const int ntzw = args.nchannels_y * args.nsamples_y; + const int ntx = args.tile_map ? args.ntx_flat : (args.ncols_max + config.J - 1) / config.J; + const int ntzw = nchannels_grid * args.nsamples_y; const dim3 block_nums_xy_tiling(nty, ntx, ntzw); GGML_ASSERT(args.nchannels_y % args.nchannels_x == 0); @@ -1421,18 +1488,71 @@ const uint3 blocks_per_ne00_fd = init_fastdiv_values(args.ncols_x / ggml_cuda_type_traits::qk); const uint3 ntx_fd = init_fastdiv_values(ntx); - const uint3 nchannels_y_fd = init_fastdiv_values(args.nchannels_y); + const uint3 nchannels_y_fd = init_fastdiv_values(nchannels_grid); const uint3 nsamples_y_fd = init_fastdiv_values(args.nsamples_y); const uint3 channel_ratio_fd = init_fastdiv_values(channel_ratio); const uint3 sample_ratio_fd = init_fastdiv_values(sample_ratio); - if (!config.stream_k) { - mul_mat_q<<>> - (args.x, args.y, args.ids_dst, args.expert_bounds, args.dst, nullptr, args.y_scale, + const auto run = [&](auto * dst, const dim3 & block_nums, float * tmp_fixup) { + using dst_t = std::remove_pointer_t; + CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q), nbytes_shared); + CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q), nbytes_shared); + mul_mat_q<<>> + (args.x, args.y, args.ids_dst, args.expert_bounds, args.tile_map, dst, tmp_fixup, args.y_scale, blocks_per_ne00_fd, args.nrows_x, args.ncols_dst, args.stride_row_x, args.ncols_y, args.nrows_dst, channel_ratio_fd, nchannels_y_fd, args.stride_channel_x, args.stride_channel_y, args.stride_channel_dst, sample_ratio_fd, nsamples_y_fd, args.stride_sample_x, args.stride_sample_y, args.stride_sample_dst, ntx_fd); + }; + + // FreeToken: the fixup kernel adds partial sums into an fp32 dst, so a bf16 dst that needs it is computed in fp32 first. + // An fp16 dst always is, unless GGML_MMQ_F16_DST builds the fp16 kernels. + const auto launch = [&](const dim3 & block_nums, const bool fixup, float * tmp_fixup) { + bool via_f32 = fixup && args.dst.type != GGML_TYPE_F32; +#ifndef GGML_MMQ_F16_DST + via_f32 = via_f32 || args.dst.type == GGML_TYPE_F16; +#endif // GGML_MMQ_F16_DST + ggml_cuda_pool_alloc dst_f32(ctx.pool(id)); + float * dst_float = args.dst.type == GGML_TYPE_F32 ? (float *) args.dst.p : nullptr; + if (via_f32) { + GGML_ASSERT(args.ids_dst || (args.nchannels_y == 1 && args.nsamples_y == 1)); + dst_float = dst_f32.alloc(args.ncols_dst * args.nrows_dst); + } + + if (dst_float) { + run(dst_float, block_nums, tmp_fixup); + } else if (args.dst.type == GGML_TYPE_BF16) { + run((nv_bfloat16 *) args.dst.p, block_nums, tmp_fixup); + } +#ifdef GGML_MMQ_F16_DST + else { + run((half *) args.dst.p, block_nums, tmp_fixup); + } +#endif // GGML_MMQ_F16_DST + + if (fixup) { + CUDA_CHECK(cudaGetLastError()); + const dim3 block_nums_fixup(block_nums.x, config.I/warp_size, 1); + const dim3 block_dims_fixup(block_dims.x, block_dims.y/2, block_dims.z); + mul_mat_q_stream_k_fixup<<>> + (args.ids_dst, args.expert_bounds, dst_float, tmp_fixup, blocks_per_ne00_fd, args.nrows_x, args.ncols_dst, + args.nrows_dst, nchannels_y_fd, args.stride_channel_dst, nsamples_y_fd, args.stride_sample_dst, + ntx_fd); + } + + if (via_f32) { + CUDA_CHECK(cudaGetLastError()); + const dim3 block_nums_store(args.ncols_dst, (args.nrows_x + 255) / 256, 1); + if (args.dst.type == GGML_TYPE_BF16) { + mmq_store_dst<<>>(dst_float, (nv_bfloat16 *) args.dst.p, args.nrows_x, args.nrows_dst); + } else { + mmq_store_dst<<>>(dst_float, (half *) args.dst.p, args.nrows_x, args.nrows_dst); + } + } + }; + + if (!config.stream_k) { + launch(block_nums_xy_tiling, false, nullptr); return; } @@ -1441,7 +1561,8 @@ const int ntiles_dst = ntx * nty * ntzw; const int tiles_nwaves = (ntiles_dst + nsm - 1) / nsm; const int tiles_efficiency_percent = 100 * ntiles_dst / (nsm*tiles_nwaves); - const dim3 block_nums_stream_k(GGML_CUDA_CC_IS_NVIDIA(cc) && tiles_efficiency_percent >= 90 ? ntiles_dst : nsm, 1, 1); + // The fixup kernel does not read tile_map, so a flat MoE grid always runs one tile per block. + const dim3 block_nums_stream_k(args.tile_map || (GGML_CUDA_CC_IS_NVIDIA(cc) && tiles_efficiency_percent >= 90) ? ntiles_dst : nsm, 1, 1); GGML_ASSERT(ntiles_dst * blocks_per_ne00_fd.z < (1 << 30)); // Assert that variable kbc will not overflow. @@ -1453,25 +1574,7 @@ tmp_fixup.alloc(block_nums_stream_k.x * config.J*config.I); } - const dim3 block_nums_fixup(block_nums_stream_k.x, config.I/warp_size, 1); - const dim3 block_dims_fixup(block_dims.x, block_dims.y/2, block_dims.z); - - mul_mat_q<<>> - (args.x, args.y, args.ids_dst, args.expert_bounds, args.dst, tmp_fixup.ptr, args.y_scale, - blocks_per_ne00_fd, args.nrows_x, args.ncols_dst, args.stride_row_x, args.ncols_y, args.nrows_dst, - channel_ratio_fd, nchannels_y_fd, args.stride_channel_x, args.stride_channel_y, args.stride_channel_dst, - sample_ratio_fd, nsamples_y_fd, args.stride_sample_x, args.stride_sample_y, args.stride_sample_dst, - ntx_fd); - - if (!fixup_needed) { - return; - } - - CUDA_CHECK(cudaGetLastError()); - mul_mat_q_stream_k_fixup<<>> - (args.ids_dst, args.expert_bounds, args.dst, tmp_fixup.ptr, blocks_per_ne00_fd, args.nrows_x, args.ncols_dst, - args.nrows_dst, nchannels_y_fd, args.stride_channel_dst, nsamples_y_fd, args.stride_sample_dst, - ntx_fd); + launch(block_nums_stream_k, fixup_needed, tmp_fixup.ptr); } template @@ -1564,6 +1667,7 @@ extern DECL_MMQ_CASE(GGML_TYPE_Q6_K); // ----------------------------------------- extern DECL_MMQ_CASE(GGML_TYPE_IQ1_S); +extern DECL_MMQ_CASE(GGML_TYPE_IQ1_M); extern DECL_MMQ_CASE(GGML_TYPE_IQ2_XXS); extern DECL_MMQ_CASE(GGML_TYPE_IQ2_XS); extern DECL_MMQ_CASE(GGML_TYPE_IQ2_S); @@ -1579,7 +1683,9 @@ // ------------------------------------------------------------------------------------------------------------------------- +// FreeToken: under ids, ncols_max > 0 bounds the tokens any one expert gets and sizes the tile grid; otherwise the grid is flat, see ggml_cuda_launch_mm_ids_tile_map void ggml_cuda_mul_mat_q( - ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst); + ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst, + int64_t ncols_max = -1, bool inactive_ids = false); bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t n_experts); --- a/ggml/src/ggml-cuda/mmvq.cu +++ b/ggml/src/ggml-cuda/mmvq.cu @@ -1,4 +1,5 @@ #include "mmvq.cuh" +#include "convert.cuh" #include "quantize.cuh" #include "unary.cuh" #include "vecdotq.cuh" @@ -596,10 +597,11 @@ return 1; } -template +// FreeToken: dst_t is float, nv_bfloat16 or half +template __launch_bounds__(calc_nwarps(type, ncols_dst, get_device_table_id(), small_k, halve_iters)*ggml_cuda_get_physical_warp_size(), 1) static __global__ void mul_mat_vec_q( - const void * vx_ptr, const void * vy_ptr, const int32_t * ids_ptr, const ggml_cuda_mm_fusion_args_device fusion, float * dst_ptr, + const void * vx_ptr, const void * vy_ptr, const int32_t * ids_ptr, const ggml_cuda_mm_fusion_args_device fusion, dst_t * dst_ptr, const uint32_t ncols_x, const uint3 nchannels_y, const uint32_t stride_row_x, const uint32_t stride_col_y, uint32_t stride_col_dst, const uint3 channel_ratio, const uint32_t stride_channel_x, const uint32_t stride_channel_y, const uint32_t stride_channel_dst, const uint3 sample_ratio, @@ -608,7 +610,7 @@ const void * GGML_CUDA_RESTRICT vx = vx_ptr; const void * GGML_CUDA_RESTRICT vy = vy_ptr; const int32_t * GGML_CUDA_RESTRICT ids = ids_ptr; - float * GGML_CUDA_RESTRICT dst = dst_ptr; + dst_t * GGML_CUDA_RESTRICT dst = dst_ptr; constexpr int qk = ggml_cuda_type_traits::qk; constexpr int qi = ggml_cuda_type_traits::qi; @@ -629,7 +631,9 @@ const uint32_t channel_dst = shared_expert ? 0 : blockIdx.y; if (shared_expert) { vx = fusion.shared_up; - dst = fusion.shared_dst; + if constexpr (std::is_same_v) { + dst = fusion.shared_dst; + } stride_col_dst = fusion.shared_stride_col_dst; } @@ -642,6 +646,15 @@ channel_y = ncols_dst == 1 && ids ? fastmodulo(channel_dst, nchannels_y) : channel_dst; sample_dst = blockIdx.z; + // FreeToken: a negative id is an inactive route (hybrid decode): zero rows, no expert read + if (ncols_dst == 1 && ids && !shared_expert && int32_t(channel_x) < 0) { + if (threadIdx.y == 0 && threadIdx.x < rows_per_cuda_block && + (rows_per_cuda_block == 1 || uint32_t(row0 + threadIdx.x) < stride_col_dst)) { + dst[sample_dst*stride_sample_dst + channel_dst*stride_channel_dst + row0 + threadIdx.x] = ggml_cuda_cast(0.0f); + } + return; + } + const uint32_t sample_x = fastdiv(sample_dst, sample_ratio); const uint32_t sample_y = sample_dst; @@ -702,10 +715,10 @@ } if constexpr (type == GGML_TYPE_NVFP4) { if (use_scale) { - x_scales = x_scale[ids ? channel_x : 0]; + x_scales = x_scale[(ids ? channel_x : 0) * fusion.x_scale_stride]; } if (use_gate_scale) { - gate_scales = gate_scale[ids ? channel_x : 0]; + gate_scales = gate_scale[(ids ? channel_x : 0) * fusion.gate_scale_stride]; } } } @@ -837,7 +850,7 @@ } } } - dst[j*stride_col_dst + i] = result; + dst[j*stride_col_dst + i] = ggml_cuda_cast(result); } } } @@ -854,11 +867,11 @@ // Grid: (ceil(nrows_x / c_rows_per_block), nchannels_dst) // Block: (warp_size, ncols_dst) - each warp handles one token independently. // No shared memory reduction needed since each warp works alone. -template +template __launch_bounds__(get_mmvq_mmid_max_batch_for_device()*ggml_cuda_get_physical_warp_size(), 1) static __global__ void mul_mat_vec_q_moe( const void * vx_ptr, const void * vy_ptr, const int32_t * ids_ptr, const ggml_cuda_mm_fusion_args_device fusion, - float * dst_ptr, + dst_t * dst_ptr, const uint32_t ncols_x, const uint3 nchannels_y, const uint32_t nrows_x, const uint32_t stride_row_x, const uint32_t stride_col_y, uint32_t stride_col_dst, const uint32_t stride_channel_x, const uint32_t stride_channel_y, const uint32_t stride_channel_dst, @@ -866,7 +879,7 @@ const void * GGML_CUDA_RESTRICT vx = vx_ptr; const void * GGML_CUDA_RESTRICT vy = vy_ptr; const int32_t * GGML_CUDA_RESTRICT ids = ids_ptr; - float * GGML_CUDA_RESTRICT dst = dst_ptr; + dst_t * GGML_CUDA_RESTRICT dst = dst_ptr; constexpr int qk = ggml_cuda_type_traits::qk; constexpr int qi = ggml_cuda_type_traits::qi; @@ -878,7 +891,9 @@ const bool shared_expert = has_fusion && fusion.shared_up && blockIdx.y == gridDim.y - 1; if (shared_expert) { vx = fusion.shared_up; - dst = fusion.shared_dst; + if constexpr (std::is_same_v) { + dst = fusion.shared_dst; + } stride_col_dst = fusion.shared_stride_col_dst; } @@ -920,6 +935,14 @@ const uint32_t channel_x = shared_expert ? 0 : ids[channel_dst + token_idx * ids_stride]; const uint32_t channel_y = fastmodulo(channel_dst, nchannels_y); + // FreeToken: a negative id is an inactive route (hybrid decode): zero rows, no expert read + if (!shared_expert && int32_t(channel_x) < 0) { + if (threadIdx.x < c_rows_per_block && (c_rows_per_block == 1 || uint32_t(row0 + threadIdx.x) < nrows_x)) { + dst[channel_dst*stride_channel_dst + token_idx*stride_col_dst + row0 + threadIdx.x] = ggml_cuda_cast(0.0f); + } + return; + } + const block_q8_1 * y = ((const block_q8_1 *) vy) + channel_y*stride_channel_y + token_idx*stride_col_y; const int kbx_offset = channel_x*stride_channel_x + row0*stride_row_x; @@ -963,7 +986,7 @@ if constexpr (type == GGML_TYPE_NVFP4) { if (x_scale) { - result *= x_scale[channel_x]; + result *= x_scale[channel_x * fusion.x_scale_stride]; } } if (x_bias) { @@ -973,7 +996,7 @@ float gate_value = tmp_gate[threadIdx.x]; if constexpr (type == GGML_TYPE_NVFP4) { if (gate_scale) { - gate_value *= gate_scale[channel_x]; + gate_value *= gate_scale[channel_x * fusion.gate_scale_stride]; } } if (gate_bias) { @@ -998,7 +1021,7 @@ } } } - dst[channel_dst*stride_channel_dst + token_idx*stride_col_dst + row0 + threadIdx.x] = result; + dst[channel_dst*stride_channel_dst + token_idx*stride_col_dst + row0 + threadIdx.x] = ggml_cuda_cast(result); } if constexpr (!has_fusion) { @@ -1020,9 +1043,9 @@ return {block_nums, block_dims}; } -template +template static void mul_mat_vec_q_switch_fusion( - const void * vx, const void * vy, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, float * dst, + const void * vx, const void * vy, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, dst_t * dst, const uint32_t ncols_x, const uint3 nchannels_y, const uint32_t stride_row_x, const uint32_t stride_col_y, const uint32_t stride_col_dst, const uint3 channel_ratio, const uint32_t stride_channel_x, const uint32_t stride_channel_y, const uint32_t stride_channel_dst, const uint3 sample_ratio, @@ -1035,7 +1058,7 @@ if constexpr (c_ncols_dst == 1) { if (has_fusion) { const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, nbytes_shared, stream); - ggml_cuda_kernel_launch(mul_mat_vec_q, launch_params, + ggml_cuda_kernel_launch(mul_mat_vec_q, launch_params, vx, vy, ids, fusion, dst, ncols_x, nchannels_y, stride_row_x, stride_col_y, stride_col_dst, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride); @@ -1046,15 +1069,15 @@ GGML_ASSERT(!has_fusion && "fusion only supported for ncols_dst=1"); const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, nbytes_shared, stream); - ggml_cuda_kernel_launch(mul_mat_vec_q, launch_params, + ggml_cuda_kernel_launch(mul_mat_vec_q, launch_params, vx, vy, ids, fusion, dst, ncols_x, nchannels_y, stride_row_x, stride_col_y, stride_col_dst, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride); } -template +template static void mul_mat_vec_q_moe_launch( - const void * vx, const void * vy, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, float * dst, + const void * vx, const void * vy, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, dst_t * dst, const uint32_t ncols_x, const uint3 nchannels_y, const uint32_t nrows_x, const uint32_t stride_row_x, const uint32_t stride_col_y, const uint32_t stride_col_dst, const uint32_t stride_channel_x, const uint32_t stride_channel_y, const uint32_t stride_channel_dst, @@ -1071,13 +1094,13 @@ fusion.x_scale != nullptr || fusion.gate_scale != nullptr; if (has_fusion) { - ggml_cuda_kernel_launch(mul_mat_vec_q_moe, launch_params, + ggml_cuda_kernel_launch(mul_mat_vec_q_moe, launch_params, vx, vy, ids, fusion, dst, ncols_x, nchannels_y, nrows_x, stride_row_x, stride_col_y, stride_col_dst, stride_channel_x, stride_channel_y, stride_channel_dst, ncols_dst, ids_stride); } else { - ggml_cuda_kernel_launch(mul_mat_vec_q_moe, launch_params, + ggml_cuda_kernel_launch(mul_mat_vec_q_moe, launch_params, vx, vy, ids, fusion, dst, ncols_x, nchannels_y, nrows_x, stride_row_x, stride_col_y, stride_col_dst, stride_channel_x, stride_channel_y, stride_channel_dst, @@ -1085,9 +1108,9 @@ } } -template +template static void mul_mat_vec_q_switch_ncols_dst( - const void * vx, const void * vy, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, float * dst, + const void * vx, const void * vy, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, dst_t * dst, const int ncols_x, const int nrows_x, const int ncols_dst, const int stride_row_x, const int stride_col_y, const int stride_col_dst, const int nchannels_x, const int nchannels_y, const int nchannels_dst, @@ -1277,8 +1300,10 @@ GGML_ABORT("fatal error"); } } + +template static void mul_mat_vec_q_switch_type( - const void * vx, const ggml_type type_x, const void * vy, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, float * dst, + const void * vx, const ggml_type type_x, const void * vy, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, dst_t * dst, const int ncols_x, const int nrows_x, const int ncols_dst, const int stride_row_x, const int stride_col_y, const int stride_col_dst, const int nchannels_x, const int nchannels_y, const int nchannels_dst, @@ -1431,9 +1456,10 @@ void ggml_cuda_mul_mat_vec_q( ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst, - const ggml_cuda_mm_fusion_args_host * fusion) { - GGML_ASSERT( src1->type == GGML_TYPE_F32); - GGML_ASSERT( dst->type == GGML_TYPE_F32); + const ggml_cuda_mm_fusion_args_host * fusion, const void * src1_q8_1_pre) { + // FreeToken: activations and results may also be bf16 or fp16 + GGML_ASSERT( src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_BF16 || src1->type == GGML_TYPE_F16); + GGML_ASSERT( dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_BF16 || dst->type == GGML_TYPE_F16); GGML_ASSERT(!ids || ids->type == GGML_TYPE_I32); // Optional, used for batched GGML_MUL_MAT_ID. GGML_TENSOR_BINARY_OP_LOCALS; @@ -1498,17 +1524,20 @@ GGML_ASSERT(!ids || fusion->gate_bias->ne[1] == src0->ne[2]); fusion_local.gate_bias = fusion->gate_bias->data; } + // FreeToken: a scale may be a strided vector, read with its element stride if (fusion->x_scale) { GGML_ASSERT(fusion->x_scale->type == GGML_TYPE_F32); - GGML_ASSERT(ggml_is_contiguous(fusion->x_scale)); + GGML_ASSERT(ggml_nelements(fusion->x_scale) == fusion->x_scale->ne[0] && fusion->x_scale->nb[0] % sizeof(float) == 0); GGML_ASSERT(ggml_nelements(fusion->x_scale) == (ids ? src0->ne[2] : 1)); fusion_local.x_scale = fusion->x_scale->data; + fusion_local.x_scale_stride = fusion->x_scale->nb[0] / sizeof(float); } if (fusion->gate_scale) { GGML_ASSERT(fusion->gate_scale->type == GGML_TYPE_F32); - GGML_ASSERT(ggml_is_contiguous(fusion->gate_scale)); + GGML_ASSERT(ggml_nelements(fusion->gate_scale) == fusion->gate_scale->ne[0] && fusion->gate_scale->nb[0] % sizeof(float) == 0); GGML_ASSERT(ggml_nelements(fusion->gate_scale) == (ids ? src0->ne[2] : 1)); fusion_local.gate_scale = fusion->gate_scale->data; + fusion_local.gate_scale_stride = fusion->gate_scale->nb[0] / sizeof(float); } fusion_local.glu_op = fusion->glu_op; fusion_local.glu_limit = fusion->glu_limit; @@ -1526,12 +1555,22 @@ } const int64_t ne10_padded = GGML_PAD(ne10, MATRIX_ROW_PADDING); - ggml_cuda_pool_alloc src1_q8_1(ctx.pool(), ne13*ne12 * ne11*ne10_padded * sizeof(block_q8_1)/QK8_1); - { + // FreeToken: src1_q8_1_pre is src1 already quantized by quantize_row_q8_1_cuda with this padding and layout + ggml_cuda_pool_alloc src1_q8_1(ctx.pool()); + const void * src1_q8 = src1_q8_1_pre; + if (src1_q8 == nullptr) { + src1_q8_1.alloc(ne13*ne12 * ne11*ne10_padded * sizeof(block_q8_1)/QK8_1); + src1_q8 = src1_q8_1.get(); const int64_t s11 = src1->nb[1] / ts_src1; const int64_t s12 = src1->nb[2] / ts_src1; const int64_t s13 = src1->nb[3] / ts_src1; - quantize_row_q8_1_cuda(src1_d, nullptr, src1_q8_1.get(), src0->type, ne10, s11, s12, s13, ne10_padded, ne11, ne12, ne13, stream); + if (src1->type == GGML_TYPE_BF16) { + quantize_row_q8_1_cuda((const nv_bfloat16 *) src1->data, nullptr, src1_q8_1.get(), src0->type, ne10, s11, s12, s13, ne10_padded, ne11, ne12, ne13, stream); + } else if (src1->type == GGML_TYPE_F16) { + quantize_row_q8_1_cuda((const half *) src1->data, nullptr, src1_q8_1.get(), src0->type, ne10, s11, s12, s13, ne10_padded, ne11, ne12, ne13, stream); + } else { + quantize_row_q8_1_cuda(src1_d, nullptr, src1_q8_1.get(), src0->type, ne10, s11, s12, s13, ne10_padded, ne11, ne12, ne13, stream); + } } const int64_t s01 = src0->nb[1] / ts_src0; @@ -1556,11 +1595,23 @@ const int64_t ids_stride = ids ? ids->nb[1] / ggml_type_size(ids->type) : 0; - mul_mat_vec_q_switch_type( - src0->data, src0->type, src1_q8_1.get(), ids_d, fusion_local, dst_d, ne00, - ne01, ncols_dst, s01, stride_col_y, stride_col_dst, - ne02, nchannels_y, nchannels_dst, s02, stride_channel_y, stride_channel_dst, - ne03, ne3, s03, s13, s3, ids_stride, stream); + const auto launch = [&](auto * dst_ptr) { + mul_mat_vec_q_switch_type( + src0->data, src0->type, src1_q8, ids_d, fusion_local, dst_ptr, ne00, + ne01, ncols_dst, s01, stride_col_y, stride_col_dst, + ne02, nchannels_y, nchannels_dst, s02, stride_channel_y, stride_channel_dst, + ne03, ne3, s03, s13, s3, ids_stride, stream); + }; + if (dst->type == GGML_TYPE_BF16) { + // FreeToken: the fused shared expert writes an fp32 shared_dst + GGML_ASSERT(!fusion || !fusion->shared_up); + launch((nv_bfloat16 *) dst->data); + } else if (dst->type == GGML_TYPE_F16) { + GGML_ASSERT(!fusion || !fusion->shared_up); + launch((half *) dst->data); + } else { + launch(dst_d); + } } void ggml_cuda_op_mul_mat_vec_q( --- a/ggml/src/ggml-cuda/mmvq.cuh +++ b/ggml/src/ggml-cuda/mmvq.cuh @@ -8,8 +8,10 @@ // based on the quantization type and GPU architecture (compute capability). int get_mmvq_mmid_max_batch(ggml_type type, int cc); +// FreeToken: src1_q8_1_pre skips the internal quantization of src1 void ggml_cuda_mul_mat_vec_q(ggml_backend_cuda_context & ctx, - const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst, const ggml_cuda_mm_fusion_args_host * fusion = nullptr); + const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst, const ggml_cuda_mm_fusion_args_host * fusion = nullptr, + const void * src1_q8_1_pre = nullptr); void ggml_cuda_op_mul_mat_vec_q( ggml_backend_cuda_context & ctx, --- a/ggml/src/ggml-cuda/quantize.cu +++ b/ggml/src/ggml-cuda/quantize.cu @@ -1,4 +1,5 @@ #include "quantize.cuh" +#include "convert.cuh" #include #if defined(BLACKWELL_MMA_AVAILABLE) @@ -50,13 +51,14 @@ #endif // CUDART_VERSION >= 12080 #endif // defined(BLACKWELL_MMA_AVAILABLE) +template __launch_bounds__(CUDA_QUANTIZE_BLOCK_SIZE, 1) static __global__ void quantize_q8_1( - const float * x_ptr, void * vy_ptr, + const src_t * x_ptr, void * vy_ptr, const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03, const int64_t ne0, const uint32_t ne1, const uint3 ne2) { ggml_cuda_pdl_lc(); - const float * GGML_CUDA_RESTRICT x = x_ptr; + const src_t * GGML_CUDA_RESTRICT x = x_ptr; void * GGML_CUDA_RESTRICT vy = vy_ptr; const int64_t i0 = (int64_t)blockDim.x*blockIdx.x + threadIdx.x; @@ -81,7 +83,7 @@ const int64_t iqs = i_cont % QK8_1; // quant index ggml_cuda_pdl_sync(); - const float xi = i0 < ne00 ? x[i03*s03 + i02*s02 + i01*s01 + i00] : 0.0f; + const float xi = i0 < ne00 ? ggml_cuda_cast(x[i03*s03 + i02*s02 + i01*s01 + i00]) : 0.0f; float amax = fabsf(xi); float sum = xi; @@ -137,6 +139,9 @@ } else { const int64_t i2 = blockIdx.y % ne2; const int64_t i3 = blockIdx.y / ne2; + if (ids && ids[blockIdx.x] < 0) { + return; // FreeToken: a compact row past the active routes (inactive routes, hybrid decode) + } const int64_t i01 = ids ? ids[blockIdx.x] : blockIdx.x; base_idx = i3 * s03 + i2 * s02 + i01 * s01; } @@ -182,6 +187,9 @@ #pragma unroll for (int slot = 0; slot < n_expert_used; ++slot) { const int64_t i = ids[(int64_t) blockIdx.x * n_expert_used + slot]; + if (i < 0) { + continue; // FreeToken: an inactive route has no compact row + } scale[i] = warp_amax[0]; } } else { @@ -308,6 +316,9 @@ #pragma unroll for (int slot = 0; slot < n_expert_used; ++slot) { const int64_t i = ids[(int64_t) blockIdx.x * n_expert_used + slot]; + if (i < 0) { + continue; // FreeToken: an inactive route has no compact row + } block_fp4_mmq * yb = y + (k_block * ne1 + i); uint32_t * yqs = reinterpret_cast(yb->qs); yqs[2 * sub + 0] = q0; @@ -374,6 +385,9 @@ } else { const int64_t i2 = blockIdx.z % ne2; const int64_t i3 = blockIdx.z / ne2; + if (ids && ids[blockIdx.x] < 0) { + return; // FreeToken: a compact row past the active routes (inactive routes, hybrid decode) + } const int64_t i01 = ids ? ids[blockIdx.x] : blockIdx.x; base_pos = i3 * s03 + i2 * s02 + i01 * s01; } @@ -427,6 +441,9 @@ #pragma unroll for (int slot = 0; slot < n_expert_used; ++slot) { const int64_t i = ids[(int64_t) blockIdx.x * n_expert_used + slot]; + if (i < 0) { + continue; // FreeToken: an inactive route has no compact row + } block_fp4_mmq * yb = y + (k_block * ne1 + i); char2 * yqs2 = (char2 *) yb->qs; if (lane_in_group == 0) { @@ -452,10 +469,29 @@ GGML_UNUSED(n_expert_used); } +// FreeToken: 4 consecutive activations as float, from fp32, bf16 or fp16 +static __device__ __forceinline__ float4 load_float4(const float * x) { + return *(const float4 *) x; +} + +static __device__ __forceinline__ float4 load_float4(const nv_bfloat16 * x) { + const uint2 v = *(const uint2 *) x; + const float2 lo = __bfloat1622float2(*(const nv_bfloat162 *) &v.x); + const float2 hi = __bfloat1622float2(*(const nv_bfloat162 *) &v.y); + return make_float4(lo.x, lo.y, hi.x, hi.y); +} + +static __device__ __forceinline__ float4 load_float4(const half * x) { + const uint2 v = *(const uint2 *) x; + const float2 lo = __half22float2(*(const half2 *) &v.x); + const float2 hi = __half22float2(*(const half2 *) &v.y); + return make_float4(lo.x, lo.y, hi.x, hi.y); +} + // scatter: grid over tokens, quantize once, write to all the token's compact rows -template +template static __global__ void quantize_mmq_q8_1( - const float * __restrict__ x, const int32_t * __restrict__ ids, void * __restrict__ vy, + const src_t * __restrict__ x, const int32_t * __restrict__ ids, void * __restrict__ vy, const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03, const int64_t ne0, const int ne1, const int ne2, const int n_expert_used) { @@ -477,18 +513,20 @@ } else { const int64_t i2 = blockIdx.z % ne2; const int64_t i3 = blockIdx.z / ne2; + if (ids && ids[blockIdx.x] < 0) { + return; // FreeToken: a compact row past the active routes (inactive routes, hybrid decode) + } const int64_t i01 = ids ? ids[blockIdx.x] : blockIdx.x; base_idx = i3*s03 + i2*s02 + i01*s01; } - const float4 * x4 = (const float4 *) x; block_q8_1_mmq * y = (block_q8_1_mmq *) vy; const int64_t k_block = i0 / QK8_1_MMQ; // column block in the channel const int64_t iqs = i0 % QK8_1_MMQ; // quant index in block // Load 4 floats per thread and calculate max. abs. value between them: - const float4 xi = i0 < ne00 ? x4[(base_idx + i00)/4] : make_float4(0.0f, 0.0f, 0.0f, 0.0f); + const float4 xi = i0 < ne00 ? load_float4(x + base_idx + i00) : make_float4(0.0f, 0.0f, 0.0f, 0.0f); float amax = fabsf(xi.x); amax = fmaxf(amax, fabsf(xi.y)); amax = fmaxf(amax, fabsf(xi.z)); @@ -526,6 +564,9 @@ int64_t ib; if constexpr (scatter) { const int64_t i = ids[(int64_t) blockIdx.x * n_expert_used + slot]; + if (i < 0) { + continue; // FreeToken: an inactive route has no compact row + } ib = k_block*ne1 + i; } else { const int64_t ib0 = blockIdx.z*((int64_t)gridDim.x*gridDim.y*blockDim.x/QK8_1); // first block of channel @@ -554,8 +595,9 @@ GGML_UNUSED(n_expert_used); } +template void quantize_row_q8_1_cuda( - const float * x, const int32_t * ids, void * vy, const ggml_type type_src0, + const src_t * x, const int32_t * ids, void * vy, const ggml_type type_src0, const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03, const int64_t ne0, const int64_t ne1, const int64_t ne2, const int64_t ne3, cudaStream_t stream) { GGML_ASSERT(!ids); @@ -567,12 +609,13 @@ const dim3 num_blocks(block_num_x, ne1, ne2*ne3); const dim3 block_size(CUDA_QUANTIZE_BLOCK_SIZE, 1, 1); const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(num_blocks, block_size, 0, stream); - ggml_cuda_kernel_launch(quantize_q8_1, launch_params, x, vy, ne00, s01, s02, s03, ne0, ne1, ne2_fastdiv); + ggml_cuda_kernel_launch(quantize_q8_1, launch_params, x, vy, ne00, s01, s02, s03, ne0, ne1, ne2_fastdiv); GGML_UNUSED(type_src0); } +template void quantize_mmq_q8_1_cuda( - const float * x, const int32_t * ids, void * vy, const ggml_type type_src0, + const src_t * x, const int32_t * ids, void * vy, const ggml_type type_src0, const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03, const int64_t ne0, const int64_t ne1, const int64_t ne2, const int64_t ne3, cudaStream_t stream) { GGML_ASSERT(ne00 % 4 == 0); @@ -584,15 +627,15 @@ const dim3 block_size(CUDA_QUANTIZE_BLOCK_SIZE_MMQ, 1, 1); switch (mmq_get_q8_1_ds_layout(type_src0)) { case MMQ_Q8_1_DS_LAYOUT_D4: - quantize_mmq_q8_1 + quantize_mmq_q8_1 <<>>(x, ids, vy, ne00, s01, s02, s03, ne0, ne1, ne2, /*n_expert_used=*/0); break; case MMQ_Q8_1_DS_LAYOUT_DS4: - quantize_mmq_q8_1 + quantize_mmq_q8_1 <<>>(x, ids, vy, ne00, s01, s02, s03, ne0, ne1, ne2, /*n_expert_used=*/0); break; case MMQ_Q8_1_DS_LAYOUT_D2S6: - quantize_mmq_q8_1 + quantize_mmq_q8_1 <<>>(x, ids, vy, ne00, s01, s02, s03, ne0, ne1, ne2, /*n_expert_used=*/0); break; default: @@ -601,8 +644,9 @@ } // scatter=true reuses the quant kernel: grid over tokens, ids = inverse map (token slot -> compact row) +template void quantize_scatter_mmq_q8_1_cuda( - const float * x, const int32_t * ids_src1_inv, void * vy, const ggml_type type_src0, + const src_t * x, const int32_t * ids_src1_inv, void * vy, const ggml_type type_src0, const int64_t ne00, const int64_t stride_token, const int64_t ne0, const int64_t n_tokens, const int64_t nrows_dst, const int n_expert_used, cudaStream_t stream) { GGML_ASSERT(ne00 % 4 == 0); @@ -613,15 +657,15 @@ const dim3 block_size(CUDA_QUANTIZE_BLOCK_SIZE_MMQ, 1, 1); switch (mmq_get_q8_1_ds_layout(type_src0)) { case MMQ_Q8_1_DS_LAYOUT_D4: - quantize_mmq_q8_1<<>>( + quantize_mmq_q8_1<<>>( x, ids_src1_inv, vy, ne00, /*s01=*/0, /*s02=*/stride_token, /*s03=*/0, ne0, /*ne1=*/(int) nrows_dst, /*ne2=*/1, n_expert_used); break; case MMQ_Q8_1_DS_LAYOUT_DS4: - quantize_mmq_q8_1<<>>( + quantize_mmq_q8_1<<>>( x, ids_src1_inv, vy, ne00, /*s01=*/0, /*s02=*/stride_token, /*s03=*/0, ne0, /*ne1=*/(int) nrows_dst, /*ne2=*/1, n_expert_used); break; case MMQ_Q8_1_DS_LAYOUT_D2S6: - quantize_mmq_q8_1<<>>( + quantize_mmq_q8_1<<>>( x, ids_src1_inv, vy, ne00, /*s01=*/0, /*s02=*/stride_token, /*s03=*/0, ne0, /*ne1=*/(int) nrows_dst, /*ne2=*/1, n_expert_used); break; default: @@ -692,3 +736,15 @@ quantize_mmq_mxfp4<<>>(x, ids, vy, ne00, s01, s02, s03, ne0, ne1, ne2, /*n_expert_used=*/0); } } + +#define FREETOKEN_QUANTIZE_INSTANCES(src_t) \ + template void quantize_row_q8_1_cuda(const src_t *, const int32_t *, void *, ggml_type, int64_t, int64_t, \ + int64_t, int64_t, int64_t, int64_t, int64_t, int64_t, cudaStream_t); \ + template void quantize_mmq_q8_1_cuda(const src_t *, const int32_t *, void *, ggml_type, int64_t, int64_t, \ + int64_t, int64_t, int64_t, int64_t, int64_t, int64_t, cudaStream_t); \ + template void quantize_scatter_mmq_q8_1_cuda(const src_t *, const int32_t *, void *, ggml_type, int64_t, \ + int64_t, int64_t, int64_t, int64_t, int, cudaStream_t); + +FREETOKEN_QUANTIZE_INSTANCES(float) +FREETOKEN_QUANTIZE_INSTANCES(nv_bfloat16) +FREETOKEN_QUANTIZE_INSTANCES(half) --- a/ggml/src/ggml-cuda/quantize.cuh +++ b/ggml/src/ggml-cuda/quantize.cuh @@ -16,13 +16,16 @@ ggml_type type_src0, int64_t ne00, int64_t s01, int64_t s02, int64_t s03, int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3, cudaStream_t stream); +// FreeToken: src_t is float, nv_bfloat16 or half +template void quantize_row_q8_1_cuda( - const float * x, const int32_t * ids, void * vy, + const src_t * x, const int32_t * ids, void * vy, ggml_type type_src0, int64_t ne00, int64_t s01, int64_t s02, int64_t s03, int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3, cudaStream_t stream); +template void quantize_mmq_q8_1_cuda( - const float * x, const int32_t * ids, void * vy, + const src_t * x, const int32_t * ids, void * vy, ggml_type type_src0, int64_t ne00, int64_t s01, int64_t s02, int64_t s03, int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3, cudaStream_t stream); @@ -57,7 +60,8 @@ int n_expert_used, cudaStream_t stream); -void quantize_scatter_mmq_q8_1_cuda(const float * x, +template +void quantize_scatter_mmq_q8_1_cuda(const src_t * x, const int32_t * ids_src1_inv, void * vy, ggml_type type_src0, --- /dev/null +++ b/ggml/src/ggml-cuda/template-instances/mmq-instance-iq1_m.cu @@ -0,0 +1,5 @@ +// FreeToken: not generated, llama.cpp has no IQ1_M MMQ. + +#include "../mmq.cuh" + +DECL_MMQ_CASE(GGML_TYPE_IQ1_M);