trending/colibri
1
0
Fork 0
mirror of https://github.com/JustVugg/colibri.git synced 2026-10-02 02:54:37 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1716 from jtinbergen/sse41-tier-upstream-next-v2

Browse Source 
perf(qwen36): int8-during-load dense weights, SIMD f16/bf16 conversion, wire slot_ensure_int8 onto #1271's unpack_int4_to_int8
This commit is contained in:
Vincenzo Fornaro
2026-09-24 21:57:29 +02:00
committed by GitHub
parent 20e4582d7d 2c7979f5a6
commit 53ca388004
12 changed files with 790 additions and 246 deletions
Download Patch File Download Diff File Expand all files Collapse all files
c/Makefile
+30 -2
View File
@@ -508,7 +508,7 @@ TEST_RULES := $(shell sed -n 's|^tests/\(test_[a-z0-9_]*\)\$$(EXE):.*|\1|p' $(f
TEST_EXCLUDE = test_uring test_deepseek_v4 test_v4_ownership test_v4_serve_framing \
test_segment_adapters_registration test_segment_adapters_real \
test_edge_adapters_registration test_edge_adapters_real \
test_gsgemv_sse41 test_qgemv_sse41 test_olmoe_dot_i8_16_sse41 test_qwen38_tier_engine \
test_gsgemv_sse41 test_qgemv_sse41 test_olmoe_dot_i8_16_sse41 test_st_f16_bf16_simd_sse41 test_qwen38_tier_engine \
test_e8x4g64_loader \
test_i4_grouped_sse41 test_i4_grouped_sse41_o1 test_i4_grouped_sse41_no_contract
TEST_BINS = $(addprefix tests/,$(addsuffix $(EXE),$(filter-out $(TEST_EXCLUDE),$(TEST_RULES))))
@@ -518,7 +518,8 @@ TEST_BINS += tests/test_exact_dot$(EXE)
endif
ifneq (,$(X86_64))
TEST_BINS += tests/test_gsgemv_sse41$(EXE) tests/test_qgemv_sse41$(EXE) \
tests/test_olmoe_dot_i8_16_sse41$(EXE) tests/test_i4_grouped_sse41$(EXE) \
tests/test_olmoe_dot_i8_16_sse41$(EXE) tests/test_st_f16_bf16_simd_sse41$(EXE) \
tests/test_i4_grouped_sse41$(EXE) \
tests/test_i4_grouped_sse41_o1$(EXE) tests/test_i4_grouped_sse41_no_contract$(EXE)
endif
ifeq ($(COLI_V4_SUPPORTED),1)
@@ -1240,6 +1241,11 @@ tests/test_qwen36_tok_truncated$(EXE): tests/test_qwen36_tok_truncated.c gsgemv.
tests/test_qwen36_encode_oom$(EXE): tests/test_qwen36_encode_oom.c gsgemv.h qgemv.h sse41_kernels.h qwen36.c expert_ffn.h qwen36_tier.h st.h json.h compat.h $(QWEN36_TIER_SRC) $(CUDA_OBJ)
$(CC) $(QWEN36_CFLAGS) $< $(QWEN36_TIER_SRC) $(CUDA_OBJ) -o $@ $(QWEN36_LDFLAGS)
# slot_ensure_int8's wiring onto #1271's unpack_int4_to_int8: regression check
# that its output matches the old separate scalar nibble-unpack loop it replaced.
tests/test_qwen36_slot_int8$(EXE): tests/test_qwen36_slot_int8.c qwen36.c qwen36_tier.h st.h json.h compat.h $(QWEN36_TIER_SRC) $(CUDA_OBJ)
$(CC) $(QWEN36_CFLAGS) $< $(QWEN36_TIER_SRC) $(CUDA_OBJ) -o $@ $(QWEN36_LDFLAGS)
# the dense trunk's integer path: quantizer contract, int4 planar packing, dispatch.
tests/test_qwen36_dense_idot$(EXE): tests/test_qwen36_dense_idot.c gsgemv.h qgemv.h sse41_kernels.h qwen36.c expert_ffn.h qwen36_tier.h st.h json.h compat.h $(QWEN36_TIER_SRC) $(CUDA_OBJ)
$(CC) $(QWEN36_CFLAGS) $< $(QWEN36_TIER_SRC) $(CUDA_OBJ) -o $@ $(QWEN36_LDFLAGS)
@@ -1381,6 +1387,28 @@ tests/test_dup_name_refusal$(EXE): tests/test_dup_name_refusal.c st.h json.h com
tests/test_st_shape$(EXE): tests/test_st_shape.c st.h json.h compat.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
# Exhaustive bit-exact gate for st.h's bf16_to_f32_bulk/f16_to_f32_bulk AVX2 tier
# (all 65536 patterns per format -- see the file for why no tolerance applies).
tests/test_st_f16_bf16_simd$(EXE): tests/test_st_f16_bf16_simd.c st.h json.h compat.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
# Same gate, forced onto the SSE4.1 tier: overrides the host's default -march
# so the SSE4.1 body in st.h is actually exercised even on a devbox that
# would otherwise always pick the AVX2 tier.
tests/test_st_f16_bf16_simd_sse41$(EXE): tests/test_st_f16_bf16_simd.c st.h json.h compat.h
$(CC) $(CFLAGS) -msse4.1 -mno-avx2 -mno-fma $< -o $@ $(LDFLAGS)
# bf16_to_f32_bulk/f16_to_f32_bulk vs the scalar per-element reference, NOT a
# test gate. Build on demand: make tests/bench_st_f16_bf16_simd ARCH=native
tests/bench_st_f16_bf16_simd$(EXE): tests/bench_st_f16_bf16_simd.c st.h json.h compat.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
# Synthetic peak-RSS proxy for qwen36.c's dense-int8-during-load change
# (load_tq vs the old post-hoc qdw_register pass), NOT a test gate. Build on
# demand: make tests/bench_load_tq_peak_rss
tests/bench_load_tq_peak_rss$(EXE): tests/bench_load_tq_peak_rss.c
$(CC) -O3 -march=native $< -o $@ -lm
tests/test_st$(EXE): tests/test_st.c st.h json.h compat.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
c/qwen36.c
+202 -188
View File
@@ -691,16 +691,35 @@ typedef struct {
int expert_down_bits, expert_down_gs;
} Cfg;
/* ---- Dense int8: a dense matrix that is quantized to int8 during load
* (load_tq, below matmul_d) instead of loaded as f32 and quantized in a
* separate pass afterward -- so the f32 staging buffer for THIS matrix alone
* is what's briefly resident, not every dense matrix in the model at once.
* `w` is the f32 copy: kept (and `q`/`sc` left NULL) when COLI_DENSE_I8=0,
* the reference/parity path; freed once `q`/`sc` are populated otherwise
* (COLI_KEEP_F32=1 keeps it alongside them, for debugging). matmul_d
* dispatches on q!=NULL directly -- no pointer-keyed scan. */
/* q/sc: int8 rows with one scale per row (the classic copy, what the VRAM
* tier uploads). q4/sg: the same matrix as int4 planar blocks of 64 with one
* scale per group (COLI_DENSE_BITS=4), the layout the K1b grouped kernel
* reads; ng = I/64 groups per row. */
typedef struct { const float *w; int8_t *q; float *sc; int I, O; uint8_t *q4; float *sg; int ng; } QW;
static void qw_free(QW *w) {
free((void*)w->w); free(w->q); free(w->sc); free(w->q4); free(w->sg);
w->w = NULL; w->q = NULL; w->sc = NULL; w->q4 = NULL; w->sg = NULL; w->ng = 0;
}
/* ---------- per-layer dense weights ---------- */
typedef struct {
float *in_ln, *post_ln, *q, *k, *v, *o, *qn, *kn, *gate, *gate_bias;
float *sh_g, *sh_u, *sh_d, *sh_gate; /* shared expert (dense f32) + shared_expert_gate */
float *in_ln, *post_ln, *qn, *kn, *gate_bias;
QW q, k, v, o, gate;
QW sh_g, sh_u, sh_d; float *sh_gate; /* shared expert (dense, int8-during-load) + shared_expert_gate */
/* Gated DeltaNet (linear_attention) dense weights (f16->f32 via st_read_f32). */
float *dn_qkv, *dn_z, *dn_b, *dn_a; /* in_proj_qkv/z/b/a */
QW dn_qkv, dn_z; float *dn_b, *dn_a; /* in_proj_qkv/z (int8-during-load), b/a (not dense-matmul'd) */
float *dn_conv; /* conv1d.weight [conv_dim, convk] (groups=conv_dim) */
float *dn_dtbias, *dn_alog; /* dt_bias[vh], A_log[vh] */
float *dn_norm; /* RMSNormGated weight [vdim] */
float *dn_out; /* out_proj [hidden, value_dim] */
QW dn_out; /* out_proj [hidden, value_dim] */
/* VRAM copies the tier placed (qt_dense handle + 1, 0 = stays on the CPU):
* the DeltaNet out_proj, the attention q/k/v/o and the shared expert's
* three matrices. Offered per layer as "dnout", "attnproj", "shexp";
@@ -731,7 +750,8 @@ typedef struct {
Cfg c;
shards S;
int quant_bits;
float *embed, *lm_head, *final_norm;
float *embed, *final_norm;
QW lm_head;
Layer *L;
LCache *cache; /* [n_layers] */
int *active_of; /* [n_layers] original->active idx (Phase 2: identity for all layers) */
@@ -1079,15 +1099,9 @@ static void matmul_qd(float *y, const float *x, const int8_t *q, const float *sc
}
/* ---- Dense int8: per-row quantized copies of the large f32 matrices.
* matmul_d dispatches via pointer lookup to matmul_q; COLI_DENSE_I8=0 falls
* back to f32 (reference path for parity tests). ~4x less memory traffic. */
#define QDW_MAX 1024
/* q/sc: int8 rows with one scale per row (the classic copy, what the VRAM
* tier uploads). q4/sg: the same matrix as int4 planar blocks of 64 with one
* scale per group (COLI_DENSE_BITS=4), the layout the K1b grouped kernel
* reads; ng = I/64 groups per row. */
static struct { const float *w; int8_t *q; float *sc; int I, O; uint8_t *q4; float *sg; int ng; } g_qdw[QDW_MAX];
static int g_qdw_n = 0;
* matmul_d dispatches directly off QW.q (no pointer-keyed scan -- see QW,
* above Layer); COLI_DENSE_I8=0 falls back to f32 (QW.w, reference path for
* parity tests). ~4x less memory traffic. */
/* ---- Dense trunk, integer dot products ------------------------------------
*
* Every dense GEMV of a token (DeltaNet projections and out_proj, attention
@@ -1187,10 +1201,16 @@ static int dense_int4_wanted(const char *tag){
}
return 0;
}
static void qdw_register_as(const float *W, int I, int O, const char *tag){
if (!W || !dense_i8_on() || g_qdw_n >= QDW_MAX) return;
/* Per-row max-abs / round-clamp int8 quantization of an in-memory f32 matrix
* W [O][I] row-major -- same math load_tq runs during streamed load, factored
* out so it can also run on a caller-owned buffer directly (tests that
* synthesize weights in memory, without a shard file to load from). Does not
* touch out->w -- the caller sets that (or leaves it, e.g. load_tq frees it
* right after). `tag` selects the int4 planar copy per dense_int4_wanted
* (NULL/COLI_DENSE_BITS!=4: skipped, out->q4 stays NULL). */
static void qw_quantize(const float *W, int I, int O, const char *tag, QW *out) {
int8_t *q = malloc((size_t)O*I); float *sc = malloc((size_t)O*sizeof(float));
if (!q || !sc) { free(q); free(sc); return; }
if (!q || !sc) { fprintf(stderr, "OOM qw_quantize\n"); exit(1); }
#pragma omp parallel for schedule(static)
for (int o = 0; o < O; o++) {
const float *r = W + (int64_t)o*I; float am = 0.f;
@@ -1199,47 +1219,45 @@ static void qdw_register_as(const float *W, int I, int O, const char *tag){
int8_t *d = q + (int64_t)o*I;
for (int i = 0; i < I; i++) { int v = (int)lrintf(r[i]*inv); if (v>127) v=127; if (v<-127) v=-127; d[i] = (int8_t)v; }
}
g_qdw[g_qdw_n].w=W; g_qdw[g_qdw_n].q=q; g_qdw[g_qdw_n].sc=sc; g_qdw[g_qdw_n].I=I; g_qdw[g_qdw_n].O=O;
g_qdw[g_qdw_n].q4=NULL; g_qdw[g_qdw_n].sg=NULL; g_qdw[g_qdw_n].ng=0;
if (dense_int4_wanted(tag) && I%64==0) {
uint8_t *q4=malloc((size_t)O*(I/2)); float *sg=malloc((size_t)O*(I/64)*sizeof(float));
out->q = q; out->sc = sc; out->I = I; out->O = O;
out->q4 = NULL; out->sg = NULL; out->ng = 0;
if (dense_int4_wanted(tag) && I % 64 == 0) {
uint8_t *q4 = malloc((size_t)O*(I/2)); float *sg = malloc((size_t)O*(I/64)*sizeof(float));
if (q4 && sg) {
pack_int4_g64_planar(W, q4, sg, O, I);
g_qdw[g_qdw_n].q4=q4; g_qdw[g_qdw_n].sg=sg; g_qdw[g_qdw_n].ng=I/64;
out->q4 = q4; out->sg = sg; out->ng = I/64;
} else { free(q4); free(sg); }
}
g_qdw_n++;
}
static void qdw_register(const float *W, int I, int O){ qdw_register_as(W, I, O, NULL); }
static void matmul_d(float *y, const float *x, const float *W, int S, int I, int O){
static void matmul_d(float *y, const float *x, const QW *w, int S, int I, int O){
#ifdef COLI_QWEN_BATCH_TEST
g_qwen_matmul_d_calls++;
#endif
for (int i = 0; i < g_qdw_n; i++) if (g_qdw[i].w == W && g_qdw[i].I == I) {
if (g_qdw[i].q4 || dense_idot_on()) {
if (w->q) {
if (w->q4 || dense_idot_on()) {
/* integer dot: the activation rows to int8 once, then the K1b
* grouped kernel (int4 planar) or the per-row int8 kernel */
int ng = I / 64;
int8_t *xq = malloc((size_t)S * I);
float *sx = malloc((size_t)S * sizeof(float));
int32_t *xsg = g_qdw[i].q4 ? malloc((size_t)S * ng * sizeof(int32_t)) : NULL;
if (xq && sx && (!g_qdw[i].q4 || xsg)) {
int32_t *xsg = w->q4 ? malloc((size_t)S * ng * sizeof(int32_t)) : NULL;
if (xq && sx && (!w->q4 || xsg)) {
for (int s = 0; s < S; s++)
sx[s] = dense_act_i8(x + (int64_t)s * I, I, xq + (int64_t)s * I, xsg ? xsg + (int64_t)s * ng : NULL);
if (g_qdw[i].q4) matmul_i4p_grouped_idot(y, xq, sx, xsg, g_qdw[i].q4, g_qdw[i].sg, S, I, O, 64);
else matmul_q_idot(y, xq, sx, g_qdw[i].q, g_qdw[i].sc, S, I, O);
if (w->q4) matmul_i4p_grouped_idot(y, xq, sx, xsg, w->q4, w->sg, S, I, O, 64);
else matmul_q_idot(y, xq, sx, w->q, w->sc, S, I, O);
free(xq); free(sx); free(xsg);
return;
}
free(xq); free(sx); free(xsg); /* out of memory: the f32 path below */
}
if (S > 1 && dense_batch_on())
matmul_q_batch(y, x, g_qdw[i].q, g_qdw[i].sc, S, I, O);
matmul_q_batch(y, x, w->q, w->sc, S, I, O);
else
for (int s = 0; s < S; s++) matmul_q(y+(int64_t)s*O, x+(int64_t)s*I, g_qdw[i].q, g_qdw[i].sc, I, O);
for (int s = 0; s < S; s++) matmul_q(y+(int64_t)s*O, x+(int64_t)s*I, w->q, w->sc, I, O);
return;
}
matmul(y, x, W, S, I, O);
matmul(y, x, w->w, S, I, O);
}
/* A dense matrix the tier placed in VRAM (handle+1 kept in the Layer, 0 = CPU):
* a device matmul, or 0 and the caller runs matmul_d as before. The
@@ -1250,22 +1268,16 @@ static inline int qtd_batch(int hp1, float *y, const float *x, int S, int I, int
static inline int qtd(int hp1, float *y, const float *x, int I, int O){
return hp1 > 0 && qt_dense_matmul(hp1 - 1, y, x, I, O);
}
/* Bytes of W's dense-i8 copy (int8 rows + per-row scales), 0 when there is
/* Bytes of w's dense-i8 copy (int8 rows + per-row scales), 0 when there is
* none (COLI_DENSE_I8=0): nothing to offer, the CPU path stands. */
static size_t qdw_bytes(const float *W){
for (int i = 0; i < g_qdw_n; i++)
if (g_qdw[i].w == W) return (size_t)g_qdw[i].I * g_qdw[i].O + (size_t)g_qdw[i].O * sizeof(float);
return 0;
static size_t qdw_bytes(const QW *w){
return w->q ? (size_t)w->I * w->O + (size_t)w->O * sizeof(float) : 0;
}
/* Upload W's dense-i8 copy to `dev`; handle+1, or 0 when it stays on the CPU. */
static int qdw_place(const float *W, int dev){
if (dev == QT_PLACE_CPU || !W) return 0;
for (int i = 0; i < g_qdw_n; i++)
if (g_qdw[i].w == W) {
int h = qt_dense_init(g_qdw[i].q, g_qdw[i].sc, g_qdw[i].I, g_qdw[i].O, dev);
return h >= 0 ? h + 1 : 0;
}
return 0;
/* Upload w's dense-i8 copy to `dev`; handle+1, or 0 when it stays on the CPU. */
static int qdw_place(const QW *w, int dev){
if (dev == QT_PLACE_CPU || !w->q) return 0;
int h = qt_dense_init(w->q, w->sc, w->I, w->O, dev);
return h >= 0 ? h + 1 : 0;
}
/* rmsnorm over a row of length D (in-place capable: out may == x).
@@ -1484,6 +1496,27 @@ static float *load_t_n(Model *m, const char *name, int64_t want) {
return p;
}
/* Dense matrix load, quantized to int8 (+ int4 planar per `tag`, see
* dense_int4_wanted) DURING loading rather than in a separate pass over the
* whole model afterward (see QW, above Layer): reads `name` (I*O elements,
* same size discipline as load_t_n), and when `quantize` && COLI_DENSE_I8 is
* on, quantizes it via qw_quantize and frees the f32 staging buffer right
* away -- so at most one dense matrix's f32 copy is ever resident at a time,
* not the whole model's. `quantize` is false for loaders that never ran
* through the old post-hoc qdw_register pass either (the Segment/Edge
* adapters build partial or auxiliary models straight off
* model_init_range/load_t_n, never main()'s dense-i8 block) -- passing it
* through keeps their f32-only behavior exactly as it was; `tag` is unused
* on that path. */
static void load_tq(Model *m, const char *name, int I, int O, int quantize, const char *tag, QW *out) {
float *p = load_t_n(m, name, (int64_t)I * O);
out->w = p; out->q = NULL; out->sc = NULL; out->I = I; out->O = O;
out->q4 = NULL; out->sg = NULL; out->ng = 0;
if (!quantize || !dense_i8_on()) return;
qw_quantize(p, I, O, tag, out);
if (getenv("COLI_KEEP_F32")) out->w = p; else { free(p); out->w = NULL; }
}
static void model_init_range(Model *m, const char *snap, int cap, int bits,
int layer_begin, int layer_end,
int load_boundaries, int allocate_state) {
@@ -1509,9 +1542,17 @@ static void model_init_range(Model *m, const char *snap, int cap, int bits,
exit(1);
}
double t0 = now_s();
/* Quantize during load only for the full-model path (main()'s static Model,
* load_boundaries=1): the Segment/Edge adapters build partial or auxiliary
* models straight off this same loop and never ran the old post-hoc
* qdw_register pass either, so gating on load_boundaries keeps their
* f32-only numerics exactly as they were. */
int quantize_dense = load_boundaries && dense_i8_on();
int qcount = 0; double qfreed = 0;
if (load_boundaries) {
m->embed = load_t_n(m, "model.embed_tokens.weight", (int64_t)c->vocab * c->hidden);
m->lm_head = load_t_n(m, "lm_head.weight", (int64_t)c->vocab * c->hidden);
m->embed = load_t_n(m, "model.embed_tokens.weight", (int64_t)c->vocab * c->hidden);
load_tq(m, "lm_head.weight", c->hidden, c->vocab, quantize_dense, "lmhead", &m->lm_head);
if (m->lm_head.q) { qcount++; qfreed += (double)c->hidden * c->vocab * sizeof(float); }
m->final_norm = load_t_n(m, "model.norm.weight", c->hidden);
}
m->L = calloc((size_t)c->n_layers, sizeof(Layer));
@@ -1521,15 +1562,19 @@ static void model_init_range(Model *m, const char *snap, int cap, int bits,
m->active_of = malloc((size_t)c->n_layers * sizeof(int));
for (int i = 0; i < c->n_layers; i++) m->active_of[i] = i;
char nm[256];
int q_out = c->q_heads * c->q_head_dim, kv_out = c->kv_heads * c->k_head_dim;
#define QCOUNT(field) do { if ((field).q) { qcount++; qfreed += (double)(field).I * (field).O * sizeof(float); } } while (0)
for (int i = layer_begin; i < layer_end; i++) {
int ai = m->active_of[i]; /* == i for Phase 2 */
Layer *l = &m->L[i];
/* input/post layernorms + MoE exist for every layer */
/* input/post layernorms exist for every layer */
#define LD(field, suffix, want) snprintf(nm,sizeof(nm),"model.layers.%d." suffix,ai); l->field = load_t_n(m,nm,(want))
LD(in_ln, "input_layernorm.weight", c->hidden);
LD(post_ln,"post_attention_layernorm.weight", c->hidden);
LD(gate, "mlp.gate.weight", (int64_t)c->n_experts * c->hidden);
#undef LD
snprintf(nm,sizeof(nm),"model.layers.%d.mlp.gate.weight", ai);
load_tq(m, nm, c->hidden, c->n_experts, quantize_dense, "router", &l->gate);
QCOUNT(l->gate);
/* q/k norms are per-head [head_dim]; only on attention layers, load if present */
if (c->has_qk_norm) {
snprintf(nm,sizeof(nm),"model.layers.%d.self_attn.q_norm.weight", ai);
@@ -1541,42 +1586,51 @@ static void model_init_range(Model *m, const char *snap, int cap, int bits,
snprintf(nm,sizeof(nm),"model.layers.%d.mlp.gate.e_score_correction_bias", ai);
if (st_has(&m->S, nm)) { l->gate_bias = falloc(c->n_experts); st_read_f32(&m->S, nm, l->gate_bias, 0); }
else l->gate_bias = NULL;
/* shared expert (dense f32) */
#define LD2(field, suffix, want) snprintf(nm,sizeof(nm),"model.layers.%d.mlp.shared_expert." suffix,ai); l->field = load_t_n(m,nm,(want))
LD2(sh_g, "gate_proj.weight", (int64_t)c->shared_inter * c->hidden);
LD2(sh_u, "up_proj.weight", (int64_t)c->shared_inter * c->hidden);
LD2(sh_d, "down_proj.weight", (int64_t)c->hidden * c->shared_inter);
#undef LD2
/* shared expert (dense, int8-during-load) */
snprintf(nm,sizeof(nm),"model.layers.%d.mlp.shared_expert.gate_proj.weight", ai);
load_tq(m, nm, c->hidden, c->shared_inter, quantize_dense, "shexp", &l->sh_g); QCOUNT(l->sh_g);
snprintf(nm,sizeof(nm),"model.layers.%d.mlp.shared_expert.up_proj.weight", ai);
load_tq(m, nm, c->hidden, c->shared_inter, quantize_dense, "shexp", &l->sh_u); QCOUNT(l->sh_u);
snprintf(nm,sizeof(nm),"model.layers.%d.mlp.shared_expert.down_proj.weight", ai);
load_tq(m, nm, c->shared_inter, c->hidden, quantize_dense, "shexp", &l->sh_d); QCOUNT(l->sh_d);
/* shared_expert_gate: Linear(hidden -> 1), sigmoid-gated shared expert */
snprintf(nm,sizeof(nm),"model.layers.%d.mlp.shared_expert_gate.weight", ai);
l->sh_gate = st_has(&m->S, nm) ? load_t_n(m, nm, c->hidden) : NULL;
if (c->is_attn[i]) {
/* Gated Attention (full_attention) layer */
#define LD3(field, suffix, want) snprintf(nm,sizeof(nm),"model.layers.%d.self_attn." suffix,ai); l->field = load_t_n(m,nm,(want))
LD3(q, "q_proj.weight", (int64_t)c->q_heads * c->q_head_dim * c->hidden);
LD3(k, "k_proj.weight", (int64_t)c->kv_heads * c->k_head_dim * c->hidden);
LD3(v, "v_proj.weight", (int64_t)c->kv_heads * c->v_head_dim * c->hidden);
LD3(o, "o_proj.weight", (int64_t)c->hidden * c->o_in);
#undef LD3
l->dn_qkv=l->dn_z=l->dn_b=l->dn_a=l->dn_conv=NULL;
l->dn_dtbias=l->dn_alog=l->dn_norm=l->dn_out=NULL;
/* Gated Attention (full_attention) layer, dense projections int8-during-load */
snprintf(nm,sizeof(nm),"model.layers.%d.self_attn.q_proj.weight", ai);
load_tq(m, nm, c->hidden, q_out, quantize_dense, "attn", &l->q); QCOUNT(l->q);
snprintf(nm,sizeof(nm),"model.layers.%d.self_attn.k_proj.weight", ai);
load_tq(m, nm, c->hidden, kv_out, quantize_dense, "attn", &l->k); QCOUNT(l->k);
snprintf(nm,sizeof(nm),"model.layers.%d.self_attn.v_proj.weight", ai);
load_tq(m, nm, c->hidden, kv_out, quantize_dense, "attn", &l->v); QCOUNT(l->v);
snprintf(nm,sizeof(nm),"model.layers.%d.self_attn.o_proj.weight", ai);
load_tq(m, nm, c->o_in, c->hidden, quantize_dense, "attn", &l->o); QCOUNT(l->o);
l->dn_qkv=l->dn_z=(QW){0}; l->dn_b=l->dn_a=l->dn_conv=NULL;
l->dn_dtbias=l->dn_alog=l->dn_norm=NULL; l->dn_out=(QW){0};
} else {
/* Gated DeltaNet (linear_attention) layer */
l->q=l->k=l->v=l->o=NULL;
l->q=l->k=l->v=l->o=(QW){0};
#define LD4(field, suffix, want) snprintf(nm,sizeof(nm),"model.layers.%d.linear_attn." suffix,ai); l->field = load_t_n(m,nm,(want))
int64_t vdim_tot = (int64_t)c->dn_vheads * c->dn_vdim;
LD4(dn_qkv, "in_proj_qkv.weight", (int64_t)c->dn_conv_dim * c->hidden);
LD4(dn_z, "in_proj_z.weight", vdim_tot * c->hidden);
snprintf(nm,sizeof(nm),"model.layers.%d.linear_attn.in_proj_qkv.weight", ai);
load_tq(m, nm, c->hidden, c->dn_conv_dim, quantize_dense, "dnproj", &l->dn_qkv); QCOUNT(l->dn_qkv);
snprintf(nm,sizeof(nm),"model.layers.%d.linear_attn.in_proj_z.weight", ai);
load_tq(m, nm, c->hidden, (int)vdim_tot, quantize_dense, "dnproj", &l->dn_z); QCOUNT(l->dn_z);
LD4(dn_b, "in_proj_b.weight", (int64_t)c->dn_vheads * c->hidden);
LD4(dn_a, "in_proj_a.weight", (int64_t)c->dn_vheads * c->hidden);
LD4(dn_conv,"conv1d.weight", (int64_t)c->dn_conv_dim * c->dn_convk);
LD4(dn_dtbias, "dt_bias", c->dn_vheads);
LD4(dn_alog,"A_log", c->dn_vheads);
LD4(dn_norm, "norm.weight", c->dn_vdim);
LD4(dn_out, "out_proj.weight", (int64_t)c->hidden * vdim_tot);
#undef LD4
snprintf(nm,sizeof(nm),"model.layers.%d.linear_attn.out_proj.weight", ai);
load_tq(m, nm, (int)vdim_tot, c->hidden, quantize_dense, "dnout", &l->dn_out); QCOUNT(l->dn_out);
}
}
#undef QCOUNT
if (quantize_dense)
fprintf(stderr, "[dense-i8] %d matrices quantized during load, %.1f GB f32 freed\n", qcount, qfreed/1073741824.0);
m->cache = calloc((size_t)c->n_layers, sizeof(LCache));
for (int i = layer_begin; i < layer_end; i++) {
m->cache[i].cap = cap;
@@ -1822,19 +1876,15 @@ static void slot_ensure_int8(Model *m, Slot *s) {
int64_t ng = (int64_t)c->inter * c->hidden, nd = (int64_t)c->hidden * c->inter;
int8_t *w = malloc((size_t)(ng + ng + nd));
if (!w) { fprintf(stderr, "OOM slot_ensure_int8\n"); exit(1); }
const uint8_t *src4[3] = { s->g4, s->u4, s->d4 };
int64_t lens[3] = { ng, ng, nd };
int8_t *dst = w;
for (int t = 0; t < 3; t++) {
const uint8_t *p = src4[t];
for (int64_t i = 0; i < lens[t]; i += 2) {
uint8_t b = p[i >> 1];
int8_t lo = (int8_t)(b & 0xF); if (lo & 8) lo -= 16;
int8_t hi = (int8_t)((b >> 4) & 0xF); if (hi & 8) hi -= 16;
dst[i] = lo; dst[i + 1] = hi;
}
dst += lens[t];
}
/* #1271's unpack_int4_to_int8 (branchless, AVX2/NEON/scalar -- see its
* definition above load_expert_merged, which already uses it for the
* container's int4 read) instead of this function's own separate scalar
* copy of the same nibble-unpack math. g4/u4/d4 are three independently
* malloc'd packed buffers here (unlike load_expert_merged's single
* contiguous `raw`), so one call per segment. */
unpack_int4_to_int8(w, s->g4, ng);
unpack_int4_to_int8(w + ng, s->u4, ng);
unpack_int4_to_int8(w + ng + ng, s->d4, nd);
s->g = w; s->u = w + ng; s->d = w + ng + ng;
}
@@ -2047,9 +2097,9 @@ static void attention(Model *m, Layer *l, int layer, float *x, int S, int pos_ba
float *vv= falloc((int64_t)S*kv_out);
/* The projections the tier placed answer from VRAM for the whole batch,
* with one backend call per matrix; unavailable handles use CPU matmul. */
if (!qtd_batch(l->qth_q, q, x, S, D, q_out)) matmul_d(q, x, l->q, S, D, q_out);
if (!qtd_batch(l->qth_k, k, x, S, D, kv_out)) matmul_d(k, x, l->k, S, D, kv_out);
if (!qtd_batch(l->qth_v, vv, x, S, D, kv_out)) matmul_d(vv, x, l->v, S, D, kv_out);
if (!qtd_batch(l->qth_q, q, x, S, D, q_out)) matmul_d(q, x, &l->q, S, D, q_out);
if (!qtd_batch(l->qth_k, k, x, S, D, kv_out)) matmul_d(k, x, &l->k, S, D, kv_out);
if (!qtd_batch(l->qth_v, vv, x, S, D, kv_out)) matmul_d(vv, x, &l->v, S, D, kv_out);
/* split q into query (first hd) and gate (next gate_dim), both per head */
float *query = falloc((int64_t)S*H*hd);
float *gate = falloc((int64_t)S*H*gate_dim);
@@ -2111,7 +2161,7 @@ static void attention(Model *m, Layer *l, int layer, float *x, int S, int pos_ba
float g = gate_dim ? gate[o] : 0.f;
ag[o] = ctx[o] * (1.f / (1.f + expf(-g)));
}
if (!qtd_batch(l->qth_o, out, ag, S, H*hd, D)) matmul_d(out, ag, l->o, S, H*hd, D);
if (!qtd_batch(l->qth_o, out, ag, S, H*hd, D)) matmul_d(out, ag, &l->o, S, H*hd, D);
free(q); free(k); free(vv); free(query); free(gate); free(ctx); free(ag);
}
@@ -2144,10 +2194,10 @@ static void qwen_shared_experts_cpu(Model *m, Layer *l, const float *x, int S,
if (B == 1) {
for (int s=0;s<S;s++) {
const float *xs=x+(int64_t)s*D;
if(!qtd(l->qth_shg,g,xs,D,I)) matmul_d(g,xs,l->sh_g,1,D,I);
if(!qtd(l->qth_shu,u,xs,D,I)) matmul_d(u,xs,l->sh_u,1,D,I);
if(!qtd(l->qth_shg,g,xs,D,I)) matmul_d(g,xs,&l->sh_g,1,D,I);
if(!qtd(l->qth_shu,u,xs,D,I)) matmul_d(u,xs,&l->sh_u,1,D,I);
for(int i=0;i<I;i++){float sv=g[i];g[i]=(sv/(1.f+expf(-sv)))*u[i];}
if(!qtd(l->qth_shd,hh,g,I,D)) matmul_d(hh,g,l->sh_d,1,I,D);
if(!qtd(l->qth_shd,hh,g,I,D)) matmul_d(hh,g,&l->sh_d,1,I,D);
float sgate=1.f;
if(l->sh_gate){float sg=0.f;for(int i=0;i<D;i++)sg+=xs[i]*l->sh_gate[i];sgate=1.f/(1.f+expf(-sg));}
float *os=out+(int64_t)s*D;
@@ -2158,10 +2208,10 @@ static void qwen_shared_experts_cpu(Model *m, Layer *l, const float *x, int S,
float *bh=falloc((int64_t)B*D);
for(int base=0;base<S;base+=B){
int rows=S-base<B?S-base:B;
matmul_d(bg,x+(int64_t)base*D,l->sh_g,rows,D,I);
matmul_d(bu,x+(int64_t)base*D,l->sh_u,rows,D,I);
matmul_d(bg,x+(int64_t)base*D,&l->sh_g,rows,D,I);
matmul_d(bu,x+(int64_t)base*D,&l->sh_u,rows,D,I);
for(int64_t q=0;q<(int64_t)rows*I;q++){float sv=bg[q];bg[q]=(sv/(1.f+expf(-sv)))*bu[q];}
matmul_d(bh,bg,l->sh_d,rows,I,D);
matmul_d(bh,bg,&l->sh_d,rows,I,D);
for(int s=0;s<rows;s++){
const float *xs=x+(int64_t)(base+s)*D;
float sgate=1.f;
@@ -2324,7 +2374,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) {
Cfg *c = &m->c; int D = c->hidden, E = c->n_experts, K = c->topk, I = c->inter;
float *logits = falloc((int64_t)S*E);
double _tr = tm_now();
matmul_d(logits, x, l->gate, S, D, E);
matmul_d(logits, x, &l->gate, S, D, E);
tm_add(S, 4, tm_now()-_tr);
if (c->has_bias && l->gate_bias) {
for (int s = 0; s < S; s++) { float *pr = logits + (int64_t)s*E; for (int e = 0; e < E; e++) pr[e] += l->gate_bias[e]; }
@@ -2429,10 +2479,10 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) {
{
double _ts2 = tm_now();
int Ish = c->shared_inter;
if (!qtd(l->qth_shg, sh, xs, D, Ish)) matmul_d(sh, xs, l->sh_g, 1, D, Ish);
if (!qtd(l->qth_shu, shu, xs, D, Ish)) matmul_d(shu, xs, l->sh_u, 1, D, Ish);
if (!qtd(l->qth_shg, sh, xs, D, Ish)) matmul_d(sh, xs, &l->sh_g, 1, D, Ish);
if (!qtd(l->qth_shu, shu, xs, D, Ish)) matmul_d(shu, xs, &l->sh_u, 1, D, Ish);
for (int i = 0; i < Ish; i++) { float sv = sh[i]; sh[i] = (sv / (1.f + expf(-sv))) * shu[i]; }
if (!qtd(l->qth_shd, shd, sh, Ish, D)) matmul_d(shd, sh, l->sh_d, 1, Ish, D);
if (!qtd(l->qth_shd, shd, sh, Ish, D)) matmul_d(shd, sh, &l->sh_d, 1, Ish, D);
float sgate = 1.f;
if (l->sh_gate) {
float sg = 0.f; const float *wg = l->sh_gate;
@@ -2537,8 +2587,8 @@ static void deltanet(Model *m, Layer *l, int layer, float *x, int S, int pos_bas
float *qkv = qkvz + (int64_t)(s % B) * proj_dim;
float *z = qkv + conv_dim;
if (!gpu_block) {
matmul_d(qkv, xs, l->dn_qkv, 1, H, conv_dim);
matmul_d(z, xs, l->dn_z, 1, H, value_dim);
matmul_d(qkv, xs, &l->dn_qkv, 1, H, conv_dim);
matmul_d(z, xs, &l->dn_z, 1, H, value_dim);
}
matmul(b, xs, l->dn_b, 1, H, vh);
matmul(a, xs, l->dn_a, 1, H, vh);
@@ -2637,7 +2687,7 @@ static void deltanet(Model *m, Layer *l, int layer, float *x, int S, int pos_bas
}
}
if (!qtd(l->qth_dnout, out + (int64_t)s * H, outr, value_dim, H))
matmul_d(out + (int64_t)s * H, outr, l->dn_out, 1, value_dim, H);
matmul_d(out + (int64_t)s * H, outr, &l->dn_out, 1, value_dim, H);
if (tm_on() && S==1){ g_dn_sub[3]+=tm_now()-_d0; }
if (layer == 0 && s == 0 && getenv("DN_DBG")) {
FILE *dbg = fopen(getenv("DN_DBG"), "wb");
@@ -2677,18 +2727,18 @@ static void trunk_offer_dense(Model *m){
Cfg *c = &m->c;
for (int i = 0; i < c->n_layers; i++) {
if (c->is_attn[i]) continue;
size_t b = qdw_bytes(m->L[i].dn_out);
size_t b = qdw_bytes(&m->L[i].dn_out);
if (b) qt_trunk_offer("dnout", i, b);
}
for (int i = 0; i < c->n_layers; i++) {
if (!c->is_attn[i]) continue;
Layer *l = &m->L[i];
size_t bq = qdw_bytes(l->q), bk = qdw_bytes(l->k), bv = qdw_bytes(l->v), bo = qdw_bytes(l->o);
size_t bq = qdw_bytes(&l->q), bk = qdw_bytes(&l->k), bv = qdw_bytes(&l->v), bo = qdw_bytes(&l->o);
if (bq && bk && bv && bo) qt_trunk_offer("attnproj", i, bq + bk + bv + bo);
}
for (int i = 0; i < c->n_layers; i++) {
Layer *l = &m->L[i];
size_t bg = qdw_bytes(l->sh_g), bu = qdw_bytes(l->sh_u), bd = qdw_bytes(l->sh_d);
size_t bg = qdw_bytes(&l->sh_g), bu = qdw_bytes(&l->sh_u), bd = qdw_bytes(&l->sh_d);
if (bg && bu && bd) qt_trunk_offer("shexp", i, bg + bu + bd);
}
}
@@ -2701,22 +2751,22 @@ static int trunk_place_dense(Model *m, double *vram_bytes){
for (int i = 0; i < c->n_layers; i++) {
Layer *l = &m->L[i];
if (!c->is_attn[i]) {
l->qth_dnout = qdw_place(l->dn_out, qt_place_of("dnout", i));
if (l->qth_dnout) { placed++; vram += (double)qdw_bytes(l->dn_out); }
l->qth_dnout = qdw_place(&l->dn_out, qt_place_of("dnout", i));
if (l->qth_dnout) { placed++; vram += (double)qdw_bytes(&l->dn_out); }
} else {
int dev = qt_place_of("attnproj", i);
l->qth_q = qdw_place(l->q, dev); l->qth_k = qdw_place(l->k, dev);
l->qth_v = qdw_place(l->v, dev); l->qth_o = qdw_place(l->o, dev);
if (l->qth_q) { placed++; vram += (double)qdw_bytes(l->q); }
if (l->qth_k) { placed++; vram += (double)qdw_bytes(l->k); }
if (l->qth_v) { placed++; vram += (double)qdw_bytes(l->v); }
if (l->qth_o) { placed++; vram += (double)qdw_bytes(l->o); }
l->qth_q = qdw_place(&l->q, dev); l->qth_k = qdw_place(&l->k, dev);
l->qth_v = qdw_place(&l->v, dev); l->qth_o = qdw_place(&l->o, dev);
if (l->qth_q) { placed++; vram += (double)qdw_bytes(&l->q); }
if (l->qth_k) { placed++; vram += (double)qdw_bytes(&l->k); }
if (l->qth_v) { placed++; vram += (double)qdw_bytes(&l->v); }
if (l->qth_o) { placed++; vram += (double)qdw_bytes(&l->o); }
}
int dev = qt_place_of("shexp", i);
l->qth_shg = qdw_place(l->sh_g, dev); l->qth_shu = qdw_place(l->sh_u, dev); l->qth_shd = qdw_place(l->sh_d, dev);
if (l->qth_shg) { placed++; vram += (double)qdw_bytes(l->sh_g); }
if (l->qth_shu) { placed++; vram += (double)qdw_bytes(l->sh_u); }
if (l->qth_shd) { placed++; vram += (double)qdw_bytes(l->sh_d); }
l->qth_shg = qdw_place(&l->sh_g, dev); l->qth_shu = qdw_place(&l->sh_u, dev); l->qth_shd = qdw_place(&l->sh_d, dev);
if (l->qth_shg) { placed++; vram += (double)qdw_bytes(&l->sh_g); }
if (l->qth_shu) { placed++; vram += (double)qdw_bytes(&l->sh_u); }
if (l->qth_shd) { placed++; vram += (double)qdw_bytes(&l->sh_d); }
}
if (vram_bytes) *vram_bytes = vram;
return placed;
@@ -2738,17 +2788,16 @@ static int trunk_probe_gpu_wins(Model *m){
if (e && *e == '0') return 1;
if (!qt_place_is_auto()) return 1;
Cfg *c = &m->c;
int qi = -1, dev = QT_PLACE_CPU;
for (int i = 0; i < c->n_layers && qi < 0; i++) {
QW *w = NULL; int dev = QT_PLACE_CPU;
for (int i = 0; i < c->n_layers && !w; i++) {
if (c->is_attn[i]) continue;
for (int j = 0; j < g_qdw_n; j++)
if (g_qdw[j].w == m->L[i].dn_qkv) { qi = j; dev = qt_place_of("dnproj", i); break; }
if (m->L[i].dn_qkv.q) { w = &m->L[i].dn_qkv; dev = qt_place_of("dnproj", i); }
}
if (qi < 0) return 1; /* dense-i8 off: nothing will be placed */
if (!w) return 1; /* dense-i8 off: nothing will be placed */
if (dev == QT_PLACE_CPU) dev = qt_place_of("lmhead", 0);
if (dev == QT_PLACE_CPU) return 1; /* nothing placed: nothing to measure */
int I = g_qdw[qi].I, O = g_qdw[qi].O;
int h = qt_dense_init(g_qdw[qi].q, g_qdw[qi].sc, I, O, dev);
int I = w->I, O = w->O;
int h = qt_dense_init(w->q, w->sc, I, O, dev);
if (h < 0) return 1; /* cannot measure: the placer's word stands */
float *x = malloc((size_t)I * sizeof(float)), *y = malloc((size_t)O * sizeof(float));
if (!x || !y) { free(x); free(y); return 1; }
@@ -2759,9 +2808,9 @@ static int trunk_probe_gpu_wins(Model *m){
double t0 = now_s();
for (int k = 0; k < 10; k++) if (!qt_dense_matmul(h, y, x, I, O)) { free(x); free(y); return 1; }
double tg = (now_s() - t0) / 10;
for (int k = 0; k < 3; k++) matmul_q(y, x, g_qdw[qi].q, g_qdw[qi].sc, I, O);
for (int k = 0; k < 3; k++) matmul_q(y, x, w->q, w->sc, I, O);
t0 = now_s();
for (int k = 0; k < 10; k++) matmul_q(y, x, g_qdw[qi].q, g_qdw[qi].sc, I, O);
for (int k = 0; k < 10; k++) matmul_q(y, x, w->q, w->sc, I, O);
double tc = (now_s() - t0) / 10;
if (tg < gpu) gpu = tg;
if (tc < cpu) cpu = tc;
@@ -2898,7 +2947,7 @@ static float *step(Model *m, const int *ids, int S, int pos_base) {
for (int p = 0; p + 1 < S; p++) {
rmsnorm_row(erow, x + (int64_t)p*D, m->final_norm, D, c->eps);
if (!qt_lmhead_matmul(elog, erow, D, c->vocab))
matmul_d(elog, erow, m->lm_head, 1, D, c->vocab);
matmul_d(elog, erow, &m->lm_head, 1, D, c->vocab);
serve_echo(g_echo_id, pos_base + p + 1, ids[p+1], elog, c->vocab, g_echo_k);
}
free(erow); free(elog);
@@ -2909,7 +2958,7 @@ static float *step(Model *m, const int *ids, int S, int pos_base) {
float *logit = falloc(c->vocab);
double _th = tm_now();
if (!qt_lmhead_matmul(logit, last, D, c->vocab))
matmul_d(logit, last, m->lm_head, 1, D, c->vocab);
matmul_d(logit, last, &m->lm_head, 1, D, c->vocab);
if (tm_on()) { tm_add(S, 5, tm_now()-_th); if (S==1) g_tm_dec_tokens++; else g_tm_pre_tokens += S; }
free(x); free(last);
if (lf) fclose(lf);
@@ -2970,7 +3019,7 @@ static void pilot_prefetch(Model *m, int lnext, const float *x, int S) {
Layer *l = &m->L[lnext];
float *nrm_x = falloc((int64_t)S * D);
for (int s = 0; s < S; s++) rmsnorm_row(nrm_x + (int64_t)s*D, x + (int64_t)s*D, l->post_ln, D, c->eps);
matmul_d(logits, nrm_x, l->gate, S, D, E); /* int8 copy (f32 may be freed) */
matmul_d(logits, nrm_x, &l->gate, S, D, E); /* int8 copy (f32 may be freed) */
free(nrm_x);
for (int s = 0; s < S; s++) {
float *pr = logits + (int64_t)s*E;
@@ -3832,36 +3881,10 @@ int main(int argc, char **argv) {
g_expert_gs = m.c.expert_gs;
if (g_expert_gs) fprintf(stderr, "[qwen36] group-scaled experts: gs=%d\n", g_expert_gs);
fprintf(stderr, "resident weights loaded in %.1fs | RSS after load: %.2f GB\n", m.dense_load_s, rss_gb());
/* quantize the large dense matrices to int8 (COLI_DENSE_I8=0 disables) */
if (dense_i8_on()) {
double tq = now_s();
Cfg *qc = &m.c; int D2 = qc->hidden;
int q_out = qc->q_heads * qc->q_head_dim, kv_out = qc->kv_heads * qc->k_head_dim;
for (int i = 0; i < qc->n_layers; i++) {
Layer *l = &m.L[i];
qdw_register_as(l->q, D2, q_out, "attn"); qdw_register_as(l->k, D2, kv_out, "attn");
qdw_register_as(l->v, D2, kv_out, "attn"); qdw_register_as(l->o, qc->o_in, D2, "attn");
qdw_register_as(l->gate, D2, qc->n_experts, "router");
qdw_register_as(l->sh_g, D2, qc->shared_inter, "shexp"); qdw_register_as(l->sh_u, D2, qc->shared_inter, "shexp");
qdw_register_as(l->sh_d, qc->shared_inter, D2, "shexp");
qdw_register_as(l->dn_qkv, D2, qc->dn_conv_dim, "dnproj");
qdw_register_as(l->dn_z, D2, qc->dn_vheads * qc->dn_vdim, "dnproj");
qdw_register_as(l->dn_out, qc->dn_vheads * qc->dn_vdim, D2, "dnout");
}
qdw_register_as(m.lm_head, D2, qc->vocab, "lmhead");
/* Free the f32 originals -- the pointers only serve as lookup keys in
* matmul_d from here on (never dereferenced again).
* COLI_KEEP_F32=1 keeps them (debug). */
double freed = 0;
if (!getenv("COLI_KEEP_F32")) {
for (int i = 0; i < g_qdw_n; i++) {
freed += (double)g_qdw[i].I * g_qdw[i].O * sizeof(float);
free((void*)g_qdw[i].w);
}
}
fprintf(stderr, "[dense-i8] %d matrices quantized in %.1f s, %.1f GB f32 freed\n",
g_qdw_n, now_s()-tq, freed/1073741824.0);
}
/* dense matrices are quantized to int8 (+ int4 planar per COLI_DENSE_BITS/
* COLI_DENSE_INT4, see dense_int4_wanted) during model_init above
* (COLI_DENSE_I8=0 disables it) -- see load_tq/QW; model_init_range
* already logged the count and freed bytes. */
/* Optional CUDA VRAM expert tier (COLI_CUDA=1): hot experts live in
* DEVICE_LOCAL memory across the configured GPUs, misses fall back to the
@@ -3897,15 +3920,11 @@ int main(int argc, char **argv) {
* No entry (dense-i8 off) means nothing to offer, and the CPU path stands. */
{
int O_qkv = m.c.dn_conv_dim, O_z = m.c.dn_vheads * m.c.dn_vdim;
for (int i = 0; i < g_qdw_n; i++)
if (g_qdw[i].w == m.lm_head)
qt_trunk_offer("lmhead", 0, (size_t)g_qdw[i].I * g_qdw[i].O + (size_t)g_qdw[i].O * sizeof(float));
if (m.lm_head.q)
qt_trunk_offer("lmhead", 0, (size_t)m.lm_head.I * m.lm_head.O + (size_t)m.lm_head.O * sizeof(float));
for (int i = 0; i < m.c.n_layers; i++) {
if (m.c.is_attn[i]) continue;
int have = 0;
for (int j = 0; j < g_qdw_n; j++)
if (g_qdw[j].w == m.L[i].dn_qkv || g_qdw[j].w == m.L[i].dn_z) have++;
if (have == 2)
if (m.L[i].dn_qkv.q && m.L[i].dn_z.q)
qt_trunk_offer("dnproj", i, (size_t)(O_qkv + O_z) * m.c.hidden + (size_t)(O_qkv + O_z) * sizeof(float));
}
trunk_offer_dense(&m); /* dnout, attnproj, shexp: the rest of the per-token dense work */
@@ -3920,13 +3939,10 @@ int main(int argc, char **argv) {
/* The placer predicted; measure before uploading a byte of trunk. */
if (!trunk_probe_gpu_wins(&m)) qt_trunk_withdraw("measured slower than the CPU");
/* R4 role split: park the dense-i8 lm_head on COLI_LMHEAD_GPU. The
* qdw entry keyed by m.lm_head holds the int8 rows + per-row scales
* the CPU path uses; the GPU applies the identical semantics. */
for (int i = 0; i < g_qdw_n; i++)
if (g_qdw[i].w == m.lm_head) {
qt_lmhead_init(g_qdw[i].q, g_qdw[i].sc, g_qdw[i].I, g_qdw[i].O);
break;
}
* QW struct on m.lm_head holds the int8 rows + per-row scales the
* CPU path uses; the GPU applies the identical semantics. */
if (m.lm_head.q)
qt_lmhead_init(m.lm_head.q, m.lm_head.sc, m.lm_head.I, m.lm_head.O);
/* R4 step 2: DeltaNet input projections, per layer, wherever
* COLI_PLACE puts them. qkv and z are both [O_x, hidden] int8 with
* per-row scales, so fusing them is a concatenation along O -- two
@@ -3940,11 +3956,8 @@ int main(int argc, char **argv) {
if (m.c.is_attn[i]) continue;
int dev = qt_place_of("dnproj", i);
if (dev == QT_PLACE_CPU) continue;
const int8_t *q1 = NULL, *q2 = NULL; const float *s1 = NULL, *s2 = NULL;
for (int j = 0; j < g_qdw_n; j++) {
if (g_qdw[j].w == m.L[i].dn_qkv) { q1 = g_qdw[j].q; s1 = g_qdw[j].sc; }
if (g_qdw[j].w == m.L[i].dn_z) { q2 = g_qdw[j].q; s2 = g_qdw[j].sc; }
}
const int8_t *q1 = m.L[i].dn_qkv.q, *q2 = m.L[i].dn_z.q;
const float *s1 = m.L[i].dn_qkv.sc, *s2 = m.L[i].dn_z.sc;
if (!q1 || !q2) continue; /* dense-i8 off: CPU path stands */
int8_t *qf = malloc((size_t)Of * Hd);
float *sf = malloc((size_t)Of * sizeof(float));
@@ -4090,12 +4103,12 @@ typedef struct {
static void qwen36_segment_layer_free(Layer *layer) {
free(layer->in_ln); free(layer->post_ln);
free(layer->q); free(layer->k); free(layer->v); free(layer->o);
free(layer->qn); free(layer->kn); free(layer->gate); free(layer->gate_bias);
free(layer->sh_g); free(layer->sh_u); free(layer->sh_d); free(layer->sh_gate);
free(layer->dn_qkv); free(layer->dn_z); free(layer->dn_b); free(layer->dn_a);
qw_free(&layer->q); qw_free(&layer->k); qw_free(&layer->v); qw_free(&layer->o);
free(layer->qn); free(layer->kn); qw_free(&layer->gate); free(layer->gate_bias);
qw_free(&layer->sh_g); qw_free(&layer->sh_u); qw_free(&layer->sh_d); free(layer->sh_gate);
qw_free(&layer->dn_qkv); qw_free(&layer->dn_z); free(layer->dn_b); free(layer->dn_a);
free(layer->dn_conv); free(layer->dn_dtbias); free(layer->dn_alog);
free(layer->dn_norm); free(layer->dn_out);
free(layer->dn_norm); qw_free(&layer->dn_out);
}
static void qwen36_segment_model_destroy(Qwen36SegmentEngine *engine) {
@@ -4483,7 +4496,7 @@ static void qwen36_edge_engine_destroy(void *engine_impl) {
Qwen36EdgeEngine *engine = (Qwen36EdgeEngine *)engine_impl;
if (!engine) return;
free(engine->model.embed);
free(engine->model.lm_head);
qw_free(&engine->model.lm_head);
free(engine->model.final_norm);
free(engine->model.c.is_attn);
st_destroy(&engine->model.S);
@@ -4522,9 +4535,10 @@ static int qwen36_edge_engine_open(
engine->model.embed = load_t_n(
&engine->model, "model.embed_tokens.weight",
(int64_t)config->vocab * config->hidden);
engine->model.lm_head = load_t_n(
&engine->model, "lm_head.weight",
(int64_t)config->vocab * config->hidden);
/* quantize=0: this engine never ran the old post-hoc qdw_register pass
* either (only main()'s static Model did), so lm_head stays f32-only here,
* exactly as before. */
load_tq(&engine->model, "lm_head.weight", config->hidden, config->vocab, 0, "lmhead", &engine->model.lm_head);
engine->model.final_norm = load_t_n(
&engine->model, "model.norm.weight", config->hidden);
engine->model.quant_bits = container_layer_is_int4(&engine->model, 0) ? 4 : 8;
@@ -4672,7 +4686,7 @@ static int qwen36_edge_select(void *engine_impl,
}
rmsnorm_row(normalized, input + (size_t)row * config->hidden,
engine->model.final_norm, config->hidden, config->eps);
matmul_d(logits, normalized, engine->model.lm_head,
matmul_d(logits, normalized, &engine->model.lm_head,
1, config->hidden, config->vocab);
if (coli_edge_argmax(logits, (uint32_t)config->vocab,
&request->token_ids[row],
@@ -4703,7 +4717,7 @@ static int qwen36_edge_logits(void *engine_impl,
rmsnorm_row(normalized, input + (size_t)row * config->hidden,
engine->model.final_norm, config->hidden, config->eps);
matmul_d(request->logits + (size_t)row * config->vocab,
normalized, engine->model.lm_head,
normalized, &engine->model.lm_head,
1, config->hidden, config->vocab);
}
free(normalized);
c/st.h
+119 -4
View File
@@ -138,6 +138,121 @@ static inline float f16_to_f32(uint16_t h) {
float f; memcpy(&f, &u, 4); return f;
}
/* ---- bulk BF16/F16 -> F32, AVX2/SSE4.1/scalar tiers --------------------
* st_read_f32/st_read_slice_f32 convert whole tensors (up to the embed/
* lm_head matrix, vocab*hidden elements) through bf16_to_f32/f16_to_f32 one
* halfword at a time; that loop is pure overhead once the read syscall is
* off the critical path. The tiers below batch it, same idea and layering
* as gsgemv.h's AVX2/SSE4.1 kernels (immintrin.h only under the ISA guard,
* sse41_kernels.h not needed here -- no FMA, no shared load helper to reuse).
*
* BF16 -> F32 is an exact zero-pad widening for every bit pattern (BF16
* shares F32's 8-bit exponent field, so there is no special-casing --
* zero, normal, subnormal, inf, NaN all take the same `<<16`), so the
* vectorized tiers cannot disagree with the scalar reference.
*
* F16 -> F32's zero/normal/inf-NaN classes are each the same closed-form
* bit algebra as the scalar reference above, just run on several lanes at
* once -- no reassociation, nothing to round, so still bit-exact. True
* subnormals (exp==0, man!=0) need the scalar reference's shift-to-normalize
* loop, which does not vectorize; those (rare in real model weights) fall
* back to f16_to_f32 per element. Exhaustive verification (all 65536
* patterns per format) lives in tests/test_st_f16_bf16_simd.c. */
#if defined(__AVX2__) || defined(__SSE4_1__)
#include <immintrin.h>
#endif
#if defined(__AVX2__)
static void bf16_to_f32_bulk(const uint16_t *src, float *dst, int64_t n) {
int64_t i = 0;
for (; i + 8 <= n; i += 8) {
__m128i h = _mm_loadu_si128((const __m128i*)(src + i));
__m256i w = _mm256_slli_epi32(_mm256_cvtepu16_epi32(h), 16);
_mm256_storeu_ps(dst + i, _mm256_castsi256_ps(w));
}
for (; i < n; i++) dst[i] = bf16_to_f32(src[i]);
}
static void f16_to_f32_bulk(const uint16_t *src, float *dst, int64_t n) {
int64_t i = 0;
const __m256i vsign_mask = _mm256_set1_epi32(0x8000);
const __m256i vexp_mask = _mm256_set1_epi32(0x1F);
const __m256i vman_mask = _mm256_set1_epi32(0x3FF);
const __m256i v112 = _mm256_set1_epi32(112);
const __m256i vinfnan_e = _mm256_set1_epi32(0x7F800000);
const __m256i vzero = _mm256_setzero_si256();
const __m256i v31 = _mm256_set1_epi32(31);
for (; i + 8 <= n; i += 8) {
__m128i h16 = _mm_loadu_si128((const __m128i*)(src + i));
__m256i h = _mm256_cvtepu16_epi32(h16);
__m256i sign = _mm256_slli_epi32(_mm256_and_si256(h, vsign_mask), 16);
__m256i exp = _mm256_and_si256(_mm256_srli_epi32(h, 10), vexp_mask);
__m256i man = _mm256_and_si256(h, vman_mask);
__m256i normal_u = _mm256_or_si256(sign, _mm256_or_si256(
_mm256_slli_epi32(_mm256_add_epi32(exp, v112), 23), _mm256_slli_epi32(man, 13)));
__m256i infnan_u = _mm256_or_si256(sign, _mm256_or_si256(vinfnan_e, _mm256_slli_epi32(man, 13)));
__m256i exp_is_zero = _mm256_cmpeq_epi32(exp, vzero);
__m256i man_is_zero = _mm256_cmpeq_epi32(man, vzero);
__m256i is_zero = _mm256_and_si256(exp_is_zero, man_is_zero);
__m256i is_subnorm = _mm256_andnot_si256(man_is_zero, exp_is_zero);
__m256i is_infnan = _mm256_cmpeq_epi32(exp, v31);
__m256i result = _mm256_blendv_epi8(normal_u, sign, is_zero);
result = _mm256_blendv_epi8(result, infnan_u, is_infnan);
_mm256_storeu_ps(dst + i, _mm256_castsi256_ps(result));
int m = _mm256_movemask_ps(_mm256_castsi256_ps(is_subnorm));
if (m) for (int k = 0; k < 8; k++) if ((m >> k) & 1) dst[i+k] = f16_to_f32(src[i+k]);
}
for (; i < n; i++) dst[i] = f16_to_f32(src[i]);
}
#elif defined(__SSE4_1__)
static void bf16_to_f32_bulk(const uint16_t *src, float *dst, int64_t n) {
int64_t i = 0;
for (; i + 4 <= n; i += 4) {
__m128i h = _mm_loadl_epi64((const __m128i*)(src + i));
__m128i w = _mm_slli_epi32(_mm_cvtepu16_epi32(h), 16);
_mm_storeu_ps(dst + i, _mm_castsi128_ps(w));
}
for (; i < n; i++) dst[i] = bf16_to_f32(src[i]);
}
static void f16_to_f32_bulk(const uint16_t *src, float *dst, int64_t n) {
int64_t i = 0;
const __m128i vsign_mask = _mm_set1_epi32(0x8000);
const __m128i vexp_mask = _mm_set1_epi32(0x1F);
const __m128i vman_mask = _mm_set1_epi32(0x3FF);
const __m128i v112 = _mm_set1_epi32(112);
const __m128i vinfnan_e = _mm_set1_epi32(0x7F800000);
const __m128i vzero = _mm_setzero_si128();
const __m128i v31 = _mm_set1_epi32(31);
for (; i + 4 <= n; i += 4) {
__m128i h16 = _mm_loadl_epi64((const __m128i*)(src + i));
__m128i h = _mm_cvtepu16_epi32(h16);
__m128i sign = _mm_slli_epi32(_mm_and_si128(h, vsign_mask), 16);
__m128i exp = _mm_and_si128(_mm_srli_epi32(h, 10), vexp_mask);
__m128i man = _mm_and_si128(h, vman_mask);
__m128i normal_u = _mm_or_si128(sign, _mm_or_si128(
_mm_slli_epi32(_mm_add_epi32(exp, v112), 23), _mm_slli_epi32(man, 13)));
__m128i infnan_u = _mm_or_si128(sign, _mm_or_si128(vinfnan_e, _mm_slli_epi32(man, 13)));
__m128i exp_is_zero = _mm_cmpeq_epi32(exp, vzero);
__m128i man_is_zero = _mm_cmpeq_epi32(man, vzero);
__m128i is_zero = _mm_and_si128(exp_is_zero, man_is_zero);
__m128i is_subnorm = _mm_andnot_si128(man_is_zero, exp_is_zero);
__m128i is_infnan = _mm_cmpeq_epi32(exp, v31);
__m128i result = _mm_blendv_epi8(normal_u, sign, is_zero);
result = _mm_blendv_epi8(result, infnan_u, is_infnan);
_mm_storeu_ps(dst + i, _mm_castsi128_ps(result));
int m = _mm_movemask_ps(_mm_castsi128_ps(is_subnorm));
if (m) for (int k = 0; k < 4; k++) if ((m >> k) & 1) dst[i+k] = f16_to_f32(src[i+k]);
}
for (; i < n; i++) dst[i] = f16_to_f32(src[i]);
}
#else
static void bf16_to_f32_bulk(const uint16_t *src, float *dst, int64_t n) {
for (int64_t i = 0; i < n; i++) dst[i] = bf16_to_f32(src[i]);
}
static void f16_to_f32_bulk(const uint16_t *src, float *dst, int64_t n) {
for (int64_t i = 0; i < n; i++) dst[i] = f16_to_f32(src[i]);
}
#endif
static int st_open_fd(shards *S, const char *path) {
for (int i = 0; i < S->nfd; i++) if (!strcmp(S->paths[i], path)) return S->fds[i];
int fd = open(path, COMPAT_O_RDONLY);
@@ -956,9 +1071,9 @@ static int64_t st_read_f32(shards *S, const char *name, float *out, int drop) {
if (t->dtype == 2) {
memcpy(out, raw, t->nbytes);
} else if (t->dtype == 0) {
uint16_t *p = (uint16_t *)raw; for (int64_t i = 0; i < t->numel; i++) out[i] = bf16_to_f32(p[i]);
bf16_to_f32_bulk((uint16_t *)raw, out, t->numel);
} else {
uint16_t *p = (uint16_t *)raw; for (int64_t i = 0; i < t->numel; i++) out[i] = f16_to_f32(p[i]);
f16_to_f32_bulk((uint16_t *)raw, out, t->numel);
}
free(raw);
if (drop) posix_fadvise(t->fd, t->off, t->nbytes, POSIX_FADV_DONTNEED);
@@ -1344,8 +1459,8 @@ static void st_read_slice_f32(shards *S, const char *name, int64_t elem_off, int
if (nb) st_pread_full(t->fd, raw, nb, boff, "pread slice"); /* dev #331: chunked + EINTR + honest short-read */
if (nb) {
if (t->dtype == 2) memcpy(out, raw, (size_t)nb);
else if (t->dtype == 0) { uint16_t *p = raw; for (int64_t i = 0; i < n_elems; i++) out[i] = bf16_to_f32(p[i]); }
else { uint16_t *p = raw; for (int64_t i = 0; i < n_elems; i++) out[i] = f16_to_f32(p[i]); }
else if (t->dtype == 0) bf16_to_f32_bulk((uint16_t *)raw, out, n_elems);
else f16_to_f32_bulk((uint16_t *)raw, out, n_elems);
}
free(raw);
if (drop && nb) posix_fadvise(t->fd, boff, nb, POSIX_FADV_DONTNEED);
c/tests/bench_load_tq_peak_rss.c
+108
View File
@@ -0,0 +1,108 @@
/* Synthetic peak-RSS proxy for the dense-int8-during-load change (QW/load_tq
* in qwen36.c). NOT a test gate, NOT an end-to-end model-load benchmark: no
* full-size Qwen3.6 checkpoint (~70 GB) is available in this environment, so
* this isolates the actual malloc/free pattern the refactor changes, at
* matrix sizes representative of Qwen3.6-35B-A3B's dense projections
* (hidden=4096-ish square matrices), repeated for a layer count comparable to
* a real model (48 dense matrices -- roughly q/k/v/o/gate/shared x 8 layers).
*
* OLD pattern (qdw_register, pre-#1271-adjacent refactor): allocate+read every
* matrix's f32 copy first, THEN make a second pass quantizing all of them --
* so all N f32 copies and all N int8 copies are resident at the same time at
* the crossover point.
* NEW pattern (load_tq): allocate+read one matrix's f32 copy, quantize it,
* free the f32 copy, move to the next -- at most one f32 copy is ever
* resident alongside the accumulating int8 copies.
*
* Reports peak RSS (Linux /proc/self/status VmHWM) for both patterns in
* separate process runs (VmHWM is monotonic non-decreasing for a process's
* lifetime, so OLD and NEW must run as separate processes, not sequentially
* in one -- an in-process "old then new" run would report the OLD pattern's
* peak forever after). Build on demand:
*
* make tests/bench_load_tq_peak_rss ARCH=native
* ./tests/bench_load_tq_peak_rss old
* ./tests/bench_load_tq_peak_rss new
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>
#include <stdint.h>
enum { HIDDEN = 4096, OUT = 4096, N_MATRICES = 48 };
static long vmhwm_kb(void) {
FILE *f = fopen("/proc/self/status", "r");
if (!f) return -1;
char line[256]; long kb = -1;
while (fgets(line, sizeof line, f)) {
if (!strncmp(line, "VmHWM:", 6)) { sscanf(line + 6, "%ld", &kb); break; }
}
fclose(f);
return kb;
}
static void fill(float *w, int64_t n, int salt) {
for (int64_t i = 0; i < n; i++) w[i] = (float)(((i * 2654435761u + salt) % 2003) - 1000) * 0.001f;
}
static void quantize_row_major(const float *w, int I, int O, int8_t *q, float *sc) {
for (int o = 0; o < O; o++) {
const float *r = w + (int64_t)o * I; float am = 0.f;
for (int i = 0; i < I; i++) { float a = fabsf(r[i]); if (a > am) am = a; }
float s = am > 1e-12f ? am / 127.f : 1.f; sc[o] = s; float inv = 1.f / s;
int8_t *d = q + (int64_t)o * I;
for (int i = 0; i < I; i++) { int v = (int)lrintf(r[i] * inv); if (v > 127) v = 127; if (v < -127) v = -127; d[i] = (int8_t)v; }
}
}
static void run_old(void) {
int64_t n = (int64_t)HIDDEN * OUT;
float **f32s = malloc(N_MATRICES * sizeof(float*));
/* pass 1: "load" every matrix's f32 copy (as model_init_range used to,
* for every dense matrix in the whole model, before any quantization) */
for (int m = 0; m < N_MATRICES; m++) {
f32s[m] = malloc((size_t)n * sizeof(float));
fill(f32s[m], n, m);
}
printf("after loading all %d f32 matrices: VmHWM=%ld MiB\n", N_MATRICES, vmhwm_kb() / 1024);
/* pass 2: quantize every matrix (as the old post-hoc qdw_register loop
* did), freeing each f32 copy only after ALL quantization is done */
int8_t **qs = malloc(N_MATRICES * sizeof(int8_t*));
float **scs = malloc(N_MATRICES * sizeof(float*));
for (int m = 0; m < N_MATRICES; m++) {
qs[m] = malloc((size_t)n);
scs[m] = malloc((size_t)OUT * sizeof(float));
quantize_row_major(f32s[m], HIDDEN, OUT, qs[m], scs[m]);
}
printf("after quantizing all (pre-free): VmHWM=%ld MiB\n", vmhwm_kb() / 1024);
for (int m = 0; m < N_MATRICES; m++) free(f32s[m]);
printf("[old pattern] peak VmHWM=%ld MiB\n", vmhwm_kb() / 1024);
}
static void run_new(void) {
int64_t n = (int64_t)HIDDEN * OUT;
int8_t **qs = malloc(N_MATRICES * sizeof(int8_t*));
float **scs = malloc(N_MATRICES * sizeof(float*));
/* load_tq: read one matrix's f32 copy, quantize it, free it, next matrix */
for (int m = 0; m < N_MATRICES; m++) {
float *w = malloc((size_t)n * sizeof(float));
fill(w, n, m);
qs[m] = malloc((size_t)n);
scs[m] = malloc((size_t)OUT * sizeof(float));
quantize_row_major(w, HIDDEN, OUT, qs[m], scs[m]);
free(w);
}
printf("[new pattern] peak VmHWM=%ld MiB\n", vmhwm_kb() / 1024);
}
int main(int argc, char **argv) {
if (argc != 2 || (strcmp(argv[1], "old") && strcmp(argv[1], "new"))) {
fprintf(stderr, "usage: %s old|new\n", argv[0]); return 2;
}
printf("N_MATRICES=%d each %dx%d f32 (%.1f MiB) -- %s pattern\n",
N_MATRICES, HIDDEN, OUT, (double)HIDDEN * OUT * 4 / 1048576.0, argv[1]);
if (!strcmp(argv[1], "old")) run_old(); else run_new();
return 0;
}
c/tests/bench_qwen36_dense_batch.c
+7 -8
View File
@@ -39,13 +39,13 @@ static double shared_run(Model *m,Layer *l,const float *x,const float *seed,
static int shared_benchmark(void) {
Model m;memset(&m,0,sizeof(m));m.c.hidden=I;m.c.shared_inter=O;
Layer l;memset(&l,0,sizeof(l));
l.sh_g=falloc((int64_t)O*I);l.sh_u=falloc((int64_t)O*I);
l.sh_d=falloc((int64_t)I*O);l.sh_gate=falloc(I);
for(int64_t i=0;i<(int64_t)O*I;i++){l.sh_g[i]=value(i,2);l.sh_u[i]=value(i,3);}
for(int64_t i=0;i<(int64_t)I*O;i++)l.sh_d[i]=value(i,4);
l.sh_g.w=falloc((int64_t)O*I);l.sh_u.w=falloc((int64_t)O*I);
l.sh_d.w=falloc((int64_t)I*O);l.sh_gate=falloc(I);
for(int64_t i=0;i<(int64_t)O*I;i++){((float*)l.sh_g.w)[i]=value(i,2);((float*)l.sh_u.w)[i]=value(i,3);}
for(int64_t i=0;i<(int64_t)I*O;i++)((float*)l.sh_d.w)[i]=value(i,4);
for(int i=0;i<I;i++)l.sh_gate[i]=value(i,5);
qdw_register(l.sh_g,I,O);qdw_register(l.sh_u,I,O);qdw_register(l.sh_d,O,I);
if(g_qdw_n!=3){fprintf(stderr,"FAIL: expected three dense-int8 copies, got %d\n",g_qdw_n);return 1;}
qw_quantize(l.sh_g.w,I,O,NULL,&l.sh_g);qw_quantize(l.sh_u.w,I,O,NULL,&l.sh_u);qw_quantize(l.sh_d.w,O,I,NULL,&l.sh_d);
if(!l.sh_g.q||!l.sh_u.q||!l.sh_d.q){fprintf(stderr,"FAIL: expected three dense-int8 copies\n");return 1;}
float *x=falloc((int64_t)S*I),*seed=falloc((int64_t)S*I);
float *a=falloc((int64_t)S*I),*b=falloc((int64_t)S*I);
for(int64_t i=0;i<(int64_t)S*I;i++){x[i]=value(i,6);seed[i]=value(i,7);}
@@ -60,8 +60,7 @@ static int shared_benchmark(void) {
printf("qwen shared int8: S=%d D=%d I=%d weights=%.1f MiB\n",S,I,O,
3.0*I*O/1048576.0);
printf("scalar %.6f s batch %.6f s speedup %.2fx calls %d -> 3\n",ts,tb,ts/tb,S*3);
for(int i=0;i<g_qdw_n;i++){free(g_qdw[i].q);free(g_qdw[i].sc);}g_qdw_n=0;
free(l.sh_g);free(l.sh_u);free(l.sh_d);free(l.sh_gate);
qw_free(&l.sh_g);qw_free(&l.sh_u);qw_free(&l.sh_d);free(l.sh_gate);
free(x);free(seed);free(a);free(b);unsetenv("QWEN_SHARED_BATCH");unsetenv("QWEN_DENSE_BATCH");
return 0;
}
c/tests/bench_st_f16_bf16_simd.c
+72
View File
@@ -0,0 +1,72 @@
/* Local A/B for st.h's bf16_to_f32_bulk/f16_to_f32_bulk vectorized tiers vs the
* scalar per-element reference (bf16_to_f32/f16_to_f32) they replace at the
* st_read_f32/st_read_slice_f32 call sites. Not a test gate (see bench_idot,
* bench_mla_simd, bench_gemv_stream for the same pattern) -- build on demand:
*
* make tests/bench_st_f16_bf16_simd ARCH=native
* ./tests/bench_st_f16_bf16_simd
*/
#include "../st.h"
#include <stdio.h>
#include <stdlib.h>
#include <time.h>
enum { N = 4 * 1024 * 1024, REPS = 5 };
static double now_s(void) {
struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts);
return (double)ts.tv_sec + (double)ts.tv_nsec * 1e-9;
}
static double scalar_bf16(const uint16_t *src, float *dst, int64_t n) {
double t0 = now_s();
for (int64_t i = 0; i < n; i++) dst[i] = bf16_to_f32(src[i]);
return now_s() - t0;
}
static double scalar_f16(const uint16_t *src, float *dst, int64_t n) {
double t0 = now_s();
for (int64_t i = 0; i < n; i++) dst[i] = f16_to_f32(src[i]);
return now_s() - t0;
}
static double bulk_bf16(const uint16_t *src, float *dst, int64_t n) {
double t0 = now_s(); bf16_to_f32_bulk(src, dst, n); return now_s() - t0;
}
static double bulk_f16(const uint16_t *src, float *dst, int64_t n) {
double t0 = now_s(); f16_to_f32_bulk(src, dst, n); return now_s() - t0;
}
int main(void) {
uint16_t *src = malloc((size_t)N * sizeof(uint16_t));
float *a = malloc((size_t)N * sizeof(float));
float *b = malloc((size_t)N * sizeof(float));
if (!src || !a || !b) { fprintf(stderr, "OOM\n"); return 2; }
/* representative mix: mostly normal-range values (real weight magnitudes),
* a scattering of exact zeros, occasional subnormal/inf/nan bit patterns --
* not the uniform all-65536-once sweep the exactness test uses. */
for (int64_t i = 0; i < N; i++) {
int64_t m = i % 97;
if (m == 0) src[i] = 0;
else if (m == 1) src[i] = 0x7C00; /* +inf */
else if (m == 2) src[i] = 0x03FF; /* max subnormal */
else src[i] = (uint16_t)((i * 2654435761u) & 0x7BFF);
}
printf("st f16/bf16 simd bench: N=%d elements (%.1f MiB src)\n", N, N * sizeof(uint16_t) / 1048576.0);
double ts = 0, tb = 0;
for (int r = 0; r < REPS; r++) {
if (r & 1) { tb += bulk_bf16(src, b, N); ts += scalar_bf16(src, a, N); }
else { ts += scalar_bf16(src, a, N); tb += bulk_bf16(src, b, N); }
}
printf("bf16->f32: scalar %.6f s bulk %.6f s speedup %.2fx\n", ts / REPS, tb / REPS, ts / tb);
ts = 0; tb = 0;
for (int r = 0; r < REPS; r++) {
if (r & 1) { tb += bulk_f16(src, b, N); ts += scalar_f16(src, a, N); }
else { ts += scalar_f16(src, a, N); tb += bulk_f16(src, b, N); }
}
printf("f16->f32: scalar %.6f s bulk %.6f s speedup %.2fx\n", ts / REPS, tb / REPS, ts / tb);
free(src); free(a); free(b);
return 0;
}
c/tests/test_qwen36_dense_batch.c
+22 -9
View File
@@ -48,21 +48,34 @@ static void one_shape(int S, int I, int O) {
free(x);free(q);free(sc);free(ref);free(got);
}
static void clear_qdw(void) {
for(int i=0;i<g_qdw_n;i++){free(g_qdw[i].q);free(g_qdw[i].sc);}
g_qdw_n=0;
static void env_set(const char *name,const char *value) {
#ifdef _WIN32
_putenv_s(name,value);
#else
setenv(name,value,1);
#endif
}
static void env_unset(const char *name) {
#ifdef _WIN32
_putenv_s(name,"");
#else
unsetenv(name);
#endif
}
static void clear_qw(Layer *l) { qw_free(&l->sh_g); qw_free(&l->sh_u); qw_free(&l->sh_d); }
static void shared_case(const char *format,int quantized) {
enum { S=12,D=64,I=32 };
Model m;memset(&m,0,sizeof(m));m.c.hidden=D;m.c.shared_inter=I;
Layer l;memset(&l,0,sizeof(l));
l.sh_g=falloc((int64_t)I*D);l.sh_u=falloc((int64_t)I*D);
l.sh_d=falloc((int64_t)D*I);l.sh_gate=falloc(D);
for(int64_t i=0;i<(int64_t)I*D;i++){l.sh_g[i]=input_value(i,2);l.sh_u[i]=input_value(i,3);}
for(int64_t i=0;i<(int64_t)D*I;i++)l.sh_d[i]=input_value(i,4);
l.sh_g.w=falloc((int64_t)I*D);l.sh_u.w=falloc((int64_t)I*D);
l.sh_d.w=falloc((int64_t)D*I);l.sh_gate=falloc(D);
l.sh_g.I=D;l.sh_g.O=I; l.sh_u.I=D;l.sh_u.O=I; l.sh_d.I=I;l.sh_d.O=D;
for(int64_t i=0;i<(int64_t)I*D;i++){((float*)l.sh_g.w)[i]=input_value(i,2);((float*)l.sh_u.w)[i]=input_value(i,3);}
for(int64_t i=0;i<(int64_t)D*I;i++)((float*)l.sh_d.w)[i]=input_value(i,4);
for(int i=0;i<D;i++)l.sh_gate[i]=input_value(i,5);
if(quantized){qdw_register(l.sh_g,D,I);qdw_register(l.sh_u,D,I);qdw_register(l.sh_d,I,D);}
if(quantized){qw_quantize(l.sh_g.w,D,I,NULL,&l.sh_g);qw_quantize(l.sh_u.w,D,I,NULL,&l.sh_u);qw_quantize(l.sh_d.w,I,D,NULL,&l.sh_d);}
float *x=falloc((int64_t)S*D),*seed=falloc((int64_t)S*D);
float *ref=falloc((int64_t)S*D),*got=falloc((int64_t)S*D);
float *g=falloc(I),*u=falloc(I),*hh=falloc(D);
@@ -88,7 +101,7 @@ static void shared_case(const char *format,int quantized) {
(unsigned long long)g_qwen_matmul_d_calls);
printf("qwen shared batch exact: format=%s S=%d calls=%d -> 3\n",format,S,S*3);
clear_qdw();free(l.sh_g);free(l.sh_u);free(l.sh_d);free(l.sh_gate);
clear_qw(&l);free(l.sh_gate);
free(x);free(seed);free(ref);free(got);free(g);free(u);free(hh);
}
c/tests/test_qwen36_dense_idot.c
+8 -7
View File
@@ -81,17 +81,18 @@ int main(void) {
for (size_t i = 0; i < (size_t)O * I; i++) W[i] = rnd();
for (size_t i = 0; i < (size_t)S * I; i++) x[i] = rnd() * 2.f;
ref_matmul(ref, x, W, S, I, O);
/* the classic path (no flags): int8 rows, f32 activations */
/* the classic path (no flags): int8 rows, f32 activations.
* dense_idot_on caches its answer: this TU reads the env once, so
* set it before the first call (matmul_d, via qw_quantize below). */
setenv("COLI_DENSE_IDOT", "0", 1); setenv("COLI_DENSE_BITS", "8", 1);
qdw_register(W, I, O);
matmul_d(y0, x, W, S, I, O);
QW w = {0}; qw_quantize(W, I, O, NULL, &w);
matmul_d(y0, x, &w, S, I, O);
double g0 = rel_gap(y0, ref, S * O);
ck(g0 < 2e-2, "classic int8 path within 2% of the f32 reference (per-row int8)");
matmul_d(y, x, W, 1, I, O);
matmul_d(y, x, &w, 1, I, O);
ck(!memcmp(y, y0, (size_t)O * sizeof(float)), "one row and the first of a batch agree on the classic path");
/* the integer path on the same int8 rows */
g_qdw_n = 0;
/* dense_idot_on caches its answer: this TU reads the env once, so set it before the first call */
qw_free(&w);
free(W); free(x); free(ref); free(y); free(y0);
}
{
/* fresh process-wide state for the flag readers: emulate by direct calls */
c/tests/test_qwen36_dnproj_batch.c
+12 -6
View File
@@ -54,13 +54,19 @@ int main(void) {
Model m={0}; Layer l={0};
m.c.hidden=H; m.c.dn_kheads=KH; m.c.dn_vheads=VH;
m.c.dn_kdim=KD; m.c.dn_vdim=VD; m.c.dn_convk=CK; m.c.dn_conv_dim=C; m.c.eps=1e-6f;
l.dn_qkv=weights(C*H); l.dn_z=weights(Z*H); l.dn_out=weights(H*Z);
/* dn_qkv/dn_z: quantized straight into the QW the Layer holds, the same
* shape load_tq uses. dn_out stays unquantized (.w only, .q == NULL) --
* the pre-QW test never registered it either, so it always took the f32
* matmul_d fallback; both deltanet() calls below share this same l.dn_out,
* so which path it takes doesn't affect the cpu/gpu comparison. */
{ float *w = weights(C*H); qw_quantize(w, H, C, NULL, &l.dn_qkv); free(w); }
{ float *w = weights(Z*H); qw_quantize(w, H, Z, NULL, &l.dn_z); free(w); }
l.dn_out.w=weights(H*Z);
l.dn_b=weights(VH*H); l.dn_a=weights(VH*H); l.dn_conv=weights(C*CK);
l.dn_alog=weights(VH); l.dn_dtbias=weights(VH); l.dn_norm=weights(VD);
qdw_register(l.dn_qkv,H,C); qdw_register(l.dn_z,H,Z);
int8_t q[O*H]; float sc[O];
memcpy(q,g_qdw[0].q,C*H); memcpy(q+C*H,g_qdw[1].q,Z*H);
memcpy(sc,g_qdw[0].sc,C*sizeof(float)); memcpy(sc+C,g_qdw[1].sc,Z*sizeof(float));
memcpy(q,l.dn_qkv.q,C*H); memcpy(q+C*H,l.dn_z.q,Z*H);
memcpy(sc,l.dn_qkv.sc,C*sizeof(float)); memcpy(sc+C,l.dn_z.sc,Z*sizeof(float));
state(&m);
enum { S=259 };
float *x=weights(S*H), cpu[S*H], gpu[S*H], rec[VH*KD*VD], conv[C*(CK-1)];
@@ -126,9 +132,9 @@ int main(void) {
coli_cuda_shutdown();
#endif
free(x); free(m.DN_rec[0]); free(m.DN_conv[0]); free(m.DN_rec); free(m.DN_conv);
free(l.dn_qkv); free(l.dn_z); free(l.dn_out); free(l.dn_b); free(l.dn_a); free(l.dn_conv);
qw_free(&l.dn_qkv); qw_free(&l.dn_z); free((void*)l.dn_out.w);
free(l.dn_b); free(l.dn_a); free(l.dn_conv);
free(l.dn_alog); free(l.dn_dtbias); free(l.dn_norm);
for(int i=0;i<g_qdw_n;i++){free(g_qdw[i].q);free(g_qdw[i].sc);} g_qdw_n=0;
printf("DeltaNet batch: %s\n",failed?"FAIL":"outputs and recurrent state match; 259 calls -> 2");
return failed!=0;
}
c/tests/test_qwen36_slot_int8.c
+103
View File
@@ -0,0 +1,103 @@
/* Regression gate for wiring slot_ensure_int8() onto #1271's unpack_int4_to_int8
* (the same branchless AVX2/NEON/scalar kernel load_expert_merged already uses
* for the container's int4 read) instead of its own separate scalar
* nibble-unpack loop. Integer nibble extraction + sign-extend is bit-exact by
* construction (no reduction, no rounding), so this compares the NEW
* slot_ensure_int8 (calling unpack_int4_to_int8) against a copy of the OLD
* scalar loop it replaced -- not just "it compiles", a real regression check
* that the two nibble-unpack formulas produce identical bytes. */
#define main qwen36_main_unused
#include "../qwen36.c"
#undef main
static int failures;
#define CHECK(cond, ...) do { if (!(cond)) { \
fprintf(stderr,"FAIL %s:%d: ",__FILE__,__LINE__); \
fprintf(stderr,__VA_ARGS__); fputc('\n',stderr); failures++; } } while (0)
/* Verbatim copy of slot_ensure_int8's pre-#1271 scalar loop (the code this
* change replaces), used here only as the regression oracle. */
static void old_unpack(int8_t *dst, const uint8_t *p, int64_t len) {
for (int64_t i = 0; i < len; i += 2) {
uint8_t b = p[i >> 1];
int8_t lo = (int8_t)(b & 0xF); if (lo & 8) lo -= 16;
int8_t hi = (int8_t)((b >> 4) & 0xF); if (hi & 8) hi -= 16;
dst[i] = lo; dst[i + 1] = hi;
}
}
static uint8_t *random_packed(int64_t nbytes, uint32_t seed) {
uint8_t *p = malloc((size_t)nbytes);
uint32_t x = seed ? seed : 1;
for (int64_t i = 0; i < nbytes; i++) {
x ^= x << 13; x ^= x >> 17; x ^= x << 5; /* xorshift32 */
p[i] = (uint8_t)x;
}
return p;
}
/* inter/hidden chosen so ng=inter*hidden and nd=hidden*inter land on both
* sides of the AVX2 16-byte (32-element) vector boundary across the three
* segments (g/u sized ng, d sized nd). ng/nd must stay even: nibble-packing
* always yields ceil(n/2) bytes, and real containers guarantee even
* inter*hidden (SIMD/GS64-block-aligned dims), so old_unpack -- a verbatim
* copy of the pre-#1271 scalar loop, kept unmodified on purpose -- was never
* written to handle an odd total (its last iteration writes dst[i+1]
* unconditionally) and shouldn't be, either: that shape cannot occur. */
static void one_case(int inter, int hidden, const char *label) {
Model m; memset(&m, 0, sizeof(m));
m.c.inter = inter; m.c.hidden = hidden;
int64_t ng = (int64_t)inter * hidden, nd = (int64_t)hidden * inter;
CHECK(ng % 2 == 0, "%s: ng must be even (nibble-packed, see one_case)", label);
if (ng % 2) return;
Slot s; memset(&s, 0, sizeof(s));
s.g4 = random_packed(ng / 2, 1001);
s.u4 = random_packed(ng / 2, 2002);
s.d4 = random_packed(nd / 2, 3003);
slot_ensure_int8(&m, &s);
CHECK(s.g != NULL, "%s: slot_ensure_int8 did not populate s->g", label);
if (!s.g) return;
int8_t *exp_g = malloc((size_t)ng), *exp_u = malloc((size_t)ng), *exp_d = malloc((size_t)nd);
old_unpack(exp_g, s.g4, ng);
old_unpack(exp_u, s.u4, ng);
old_unpack(exp_d, s.d4, nd);
CHECK(!memcmp(s.g, exp_g, (size_t)ng), "%s: gate segment differs from old scalar loop", label);
CHECK(!memcmp(s.u, exp_u, (size_t)ng), "%s: up segment differs from old scalar loop", label);
CHECK(!memcmp(s.d, exp_d, (size_t)nd), "%s: down segment differs from old scalar loop", label);
printf("qwen36 slot_ensure_int8 exact: %s inter=%d hidden=%d (ng=%lld nd=%lld)\n",
label, inter, hidden, (long long)ng, (long long)nd);
free(exp_g); free(exp_u); free(exp_d);
free(s.g); /* s.u/s.d point into the same block as s.g (slot_ensure_int8's `w`) */
free(s.g4); free(s.u4); free(s.d4);
}
int main(void) {
one_case(4, 8, "tiny, well under one vector"); /* ng=32, nd=32 */
one_case(16, 16, "exact AVX2 vector boundary"); /* ng=256, nd=256, /16=16 (mult of 16-byte block) */
one_case(17, 16, "vector body + scalar tail"); /* ng=272 -> nb=136, not mult of 16 */
one_case(37, 42, "odd, real-expert-shaped"); /* ng=1554, nd=1554 (even: see one_case) */
one_case(2048, 512, "large, real Qwen3.6 expert scale"); /* ng=1048576, nd=1048576 */
/* re-entrancy guard: slot_ensure_int8 must no-op when s->g is already set
* (the "rematerialize only if evicted" contract) -- unrelated to which
* unpack kernel runs, but worth pinning since this function was touched */
{
Model m; memset(&m, 0, sizeof(m)); m.c.inter = 4; m.c.hidden = 8;
Slot s; memset(&s, 0, sizeof(s));
s.g4 = random_packed(16, 42); s.u4 = random_packed(16, 43); s.d4 = random_packed(16, 44);
slot_ensure_int8(&m, &s);
int8_t *g_before = s.g;
slot_ensure_int8(&m, &s); /* should be a no-op: s->g already non-NULL */
CHECK(s.g == g_before, "slot_ensure_int8 re-ran when s->g was already set");
free(s.g); free(s.g4); free(s.u4); free(s.d4);
}
if (failures) { fprintf(stderr, "qwen36 slot_ensure_int8: %d failure(s)\n", failures); return 1; }
puts("qwen36 slot_ensure_int8: ok");
return 0;
}
c/tests/test_qwen36_trunk_dense.c
+31 -22
View File
@@ -50,10 +50,10 @@ static float *rnd_matrix(int rows, int cols) {
* matmul_d. The fake backend computes exactly what a real one does with these
* bytes (x . int8 row, times the row scale), so the two must agree to float
* accumulation order. */
static double gemv_gap(int hp1, const float *W, int I, int O, int *served) {
static double gemv_gap(int hp1, const QW *w, int I, int O, int *served) {
float *x = rnd_matrix(1, I), *ya = calloc((size_t)O, sizeof(float)), *yb = calloc((size_t)O, sizeof(float));
*served = qtd(hp1, ya, x, I, O);
matmul_d(yb, x, W, 1, I, O);
matmul_d(yb, x, w, 1, I, O);
double worst = 0, scale = 1e-6;
for (int o = 0; o < O; o++) {
double d = fabs((double)ya[o] - yb[o]);
@@ -64,6 +64,15 @@ static double gemv_gap(int hp1, const float *W, int I, int O, int *served) {
return worst / scale;
}
/* Quantize a freshly-built random matrix straight into *out (the QW the
* Layer holds), then discard the f32 staging buffer -- the same "quantize
* once, free the f32 copy" shape load_tq uses. */
static void register_qw(int rows, int cols, QW *out) {
float *w = rnd_matrix(rows, cols);
qw_quantize(w, cols, rows, NULL, out);
free(w);
}
static void build(Model *m) {
memset(m, 0, sizeof *m);
Cfg *c = &m->c;
@@ -73,20 +82,20 @@ static void build(Model *m) {
c->is_attn = calloc(NL, 1); c->is_attn[1] = 1;
m->L = calloc(NL, sizeof(Layer));
Layer *dn = &m->L[0], *at = &m->L[1];
dn->dn_out = rnd_matrix(D, VH * VD); qdw_register(dn->dn_out, VH * VD, D);
dn->sh_g = rnd_matrix(SH, D); qdw_register(dn->sh_g, D, SH);
dn->sh_u = rnd_matrix(SH, D); qdw_register(dn->sh_u, D, SH);
dn->sh_d = rnd_matrix(D, SH); qdw_register(dn->sh_d, SH, D);
at->q = rnd_matrix(QH * QD, D); qdw_register(at->q, D, QH * QD);
at->k = rnd_matrix(KVH * KD, D); qdw_register(at->k, D, KVH * KD);
at->v = rnd_matrix(KVH * KD, D); qdw_register(at->v, D, KVH * KD);
at->o = rnd_matrix(D, QH * KD); qdw_register(at->o, QH * KD, D);
register_qw(D, VH * VD, &dn->dn_out);
register_qw(SH, D, &dn->sh_g);
register_qw(SH, D, &dn->sh_u);
register_qw(D, SH, &dn->sh_d);
register_qw(QH * QD, D, &at->q);
register_qw(KVH * KD, D, &at->k);
register_qw(KVH * KD, D, &at->v);
register_qw(D, QH * KD, &at->o);
/* the attention layer's shared expert is INCOMPLETE on purpose: gate and
* up have a dense-i8 copy, down does not (never registered), so "shexp"
* for layer 1 must not be offered and its three handles must stay 0 */
at->sh_g = rnd_matrix(SH, D); qdw_register(at->sh_g, D, SH);
at->sh_u = rnd_matrix(SH, D); qdw_register(at->sh_u, D, SH);
at->sh_d = rnd_matrix(D, SH); /* no qdw_register */
register_qw(SH, D, &at->sh_g);
register_qw(SH, D, &at->sh_u);
/* at->sh_d: left zeroed (q == NULL) -- no dense-i8 copy, never registered */
}
int main(void) {
@@ -109,17 +118,17 @@ int main(void) {
trunk_offer_dense(&m);
/* dnout(layer 0) + attnproj(layer 1) + shexp(layer 0); NOT shexp(layer 1) */
ck(G_offer_n - before == 3, "three components offered: dnout, attnproj, shexp of the complete layer only");
size_t want_attn = qdw_bytes(at->q) + qdw_bytes(at->k) + qdw_bytes(at->v) + qdw_bytes(at->o);
size_t want_attn = qdw_bytes(&at->q) + qdw_bytes(&at->k) + qdw_bytes(&at->v) + qdw_bytes(&at->o);
int seen_attn = 0, seen_dnout = 0, seen_shexp1 = 0;
for (int o = before; o < G_offer_n; o++) {
if (!strcmp(G_offer[o].name, "attnproj") && G_offer[o].layer == 1 && G_offer[o].bytes == want_attn) seen_attn = 1;
if (!strcmp(G_offer[o].name, "dnout") && G_offer[o].layer == 0 && G_offer[o].bytes == qdw_bytes(dn->dn_out)) seen_dnout = 1;
if (!strcmp(G_offer[o].name, "dnout") && G_offer[o].layer == 0 && G_offer[o].bytes == qdw_bytes(&dn->dn_out)) seen_dnout = 1;
if (!strcmp(G_offer[o].name, "shexp") && G_offer[o].layer == 1) seen_shexp1 = 1;
}
ck(seen_dnout, "dnout offered for the DeltaNet layer with the bytes of its dense-i8 copy");
ck(seen_attn, "attnproj offered for the attention layer as the sum of q, k, v, o");
ck(!seen_shexp1, "a shared expert missing one dense-i8 copy is not offered");
ck(qdw_bytes(at->sh_d) == 0, "a matrix without a dense-i8 copy reports zero bytes");
ck(qdw_bytes(&at->sh_d) == 0, "a matrix without a dense-i8 copy reports zero bytes");
printf("placement\n");
ck(qt_init(NL, NE, D, IH, NE, 1, 0, 1), "tier starts (int4 mode, cap == n_experts)");
@@ -129,7 +138,7 @@ int main(void) {
double vram = 0;
int placed = trunk_place_dense(&m, &vram);
ck(placed == 8, "eight matrices placed: dnout, q k v o, and the complete shared expert");
ck(vram == (double)(qdw_bytes(dn->dn_out) + want_attn + qdw_bytes(dn->sh_g) + qdw_bytes(dn->sh_u) + qdw_bytes(dn->sh_d)),
ck(vram == (double)(qdw_bytes(&dn->dn_out) + want_attn + qdw_bytes(&dn->sh_g) + qdw_bytes(&dn->sh_u) + qdw_bytes(&dn->sh_d)),
"placed bytes are the sum of the dense-i8 copies uploaded");
ck(fake_uploads == 8, "one upload per placed matrix");
ck(dn->qth_dnout > 0 && at->qth_q > 0 && at->qth_k > 0 && at->qth_v > 0 && at->qth_o > 0, "handles kept in the Layer");
@@ -137,11 +146,11 @@ int main(void) {
ck(at->qth_shg == 0 && at->qth_shu == 0 && at->qth_shd == 0, "no handle for the shared expert that was not offered");
printf("GEMV from VRAM == matmul_d on the CPU\n");
struct { const char *name; int hp1; const float *w; int I, O; } mats[] = {
{"dn_out", dn->qth_dnout, dn->dn_out, VH * VD, D},
{"q", at->qth_q, at->q, D, QH * QD}, {"k", at->qth_k, at->k, D, KVH * KD},
{"v", at->qth_v, at->v, D, KVH * KD}, {"o", at->qth_o, at->o, QH * KD, D},
{"sh_g", dn->qth_shg, dn->sh_g, D, SH}, {"sh_u", dn->qth_shu, dn->sh_u, D, SH}, {"sh_d", dn->qth_shd, dn->sh_d, SH, D},
struct { const char *name; int hp1; const QW *w; int I, O; } mats[] = {
{"dn_out", dn->qth_dnout, &dn->dn_out, VH * VD, D},
{"q", at->qth_q, &at->q, D, QH * QD}, {"k", at->qth_k, &at->k, D, KVH * KD},
{"v", at->qth_v, &at->v, D, KVH * KD}, {"o", at->qth_o, &at->o, QH * KD, D},
{"sh_g", dn->qth_shg, &dn->sh_g, D, SH}, {"sh_u", dn->qth_shu, &dn->sh_u, D, SH}, {"sh_d", dn->qth_shd, &dn->sh_d, SH, D},
};
for (size_t i = 0; i < sizeof mats / sizeof mats[0]; i++) {
int served = 0;
c/tests/test_st_f16_bf16_simd.c
+76
View File
@@ -0,0 +1,76 @@
/* Exhaustive bit-exact gate for st.h's bf16_to_f32_bulk / f16_to_f32_bulk
* (AVX2/SSE4.1/scalar tiers). BF16->F32 and F16->F32 are lossless IEEE-754
* widenings: for every one of the 65536 possible input bit patterns -- every
* normal and subnormal exponent, +-0, +-inf, and every NaN payload -- there
* is exactly one correct output, so there is no tolerance to allow: the
* vectorized tiers must produce bit-identical floats to the scalar reference
* (bf16_to_f32/f16_to_f32) for the entire input space, not just a sample.
*
* Runs the full 65536-value sweep (which lands on an exact multiple of both
* the AVX2 8-wide and SSE4.1 4-wide vector width) and a set of odd-length,
* odd-offset slices of the same data so the scalar remainder tail in each
* tier's loop is also exercised, not just the vectorized body. */
#include "../st.h"
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
static int failures;
#define CHECK(cond, ...) do { if (!(cond)) { \
fprintf(stderr, "FAIL %s:%d: ", __FILE__, __LINE__); \
fprintf(stderr, __VA_ARGS__); fputc('\n', stderr); failures++; } } while (0)
static int32_t bits_of(float f) { int32_t u; memcpy(&u, &f, 4); return u; }
static void check_range(const uint16_t *src, int64_t n, const char *label) {
float *bulk_bf = malloc((size_t)n * sizeof(float));
float *bulk_f16 = malloc((size_t)n * sizeof(float));
if (!bulk_bf || !bulk_f16) { fprintf(stderr, "OOM\n"); exit(2); }
bf16_to_f32_bulk(src, bulk_bf, n);
f16_to_f32_bulk(src, bulk_f16, n);
int64_t bf_diffs = 0, f16_diffs = 0;
int64_t bf_first = -1, f16_first = -1;
for (int64_t i = 0; i < n; i++) {
float ref_bf = bf16_to_f32(src[i]);
float ref_f16 = f16_to_f32(src[i]);
if (bits_of(ref_bf) != bits_of(bulk_bf[i])) { bf_diffs++; if (bf_first < 0) bf_first = i; }
if (bits_of(ref_f16) != bits_of(bulk_f16[i])) { f16_diffs++; if (f16_first < 0) f16_first = i; }
}
if (bf_diffs) fprintf(stderr, " bf16 first mismatch idx=%lld h=0x%04x scalar=0x%08x bulk=0x%08x\n",
(long long)bf_first, src[bf_first], bits_of(bf16_to_f32(src[bf_first])), bits_of(bulk_bf[bf_first]));
if (f16_diffs) fprintf(stderr, " f16 first mismatch idx=%lld h=0x%04x scalar=0x%08x bulk=0x%08x\n",
(long long)f16_first, src[f16_first], bits_of(f16_to_f32(src[f16_first])), bits_of(bulk_f16[f16_first]));
CHECK(bf_diffs == 0, "%s: bf16_to_f32_bulk %lld/%lld mismatches", label, (long long)bf_diffs, (long long)n);
CHECK(f16_diffs == 0, "%s: f16_to_f32_bulk %lld/%lld mismatches", label, (long long)f16_diffs, (long long)n);
free(bulk_bf); free(bulk_f16);
}
int main(void) {
/* every possible 16-bit pattern, once each -- covers +-0, every normal
* and subnormal exponent, +-inf, and every NaN payload for both formats */
uint16_t *all = malloc(65536 * sizeof(uint16_t));
if (!all) { fprintf(stderr, "OOM\n"); return 2; }
for (int32_t h = 0; h <= 0xFFFF; h++) all[h] = (uint16_t)h;
check_range(all, 65536, "full sweep");
printf("st f16/bf16 simd: exhaustive 65536/65536 patterns exact\n");
/* odd offsets/lengths so the scalar remainder tail (n%8 for AVX2, n%4 for
* SSE4.1) actually runs, on real (not synthetic-zero) data */
static const int64_t offs[] = {0, 1, 2, 3, 5, 7, 9, 13, 15, 16, 17, 100, 12345};
static const int64_t lens[] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 15, 16, 17, 65423, 65535};
for (size_t oi = 0; oi < sizeof(offs)/sizeof(offs[0]); oi++) {
for (size_t li = 0; li < sizeof(lens)/sizeof(lens[0]); li++) {
int64_t off = offs[oi], len = lens[li];
if (off + len > 65536) continue;
char label[64]; snprintf(label, sizeof label, "off=%lld len=%lld", (long long)off, (long long)len);
check_range(all + off, len, label);
}
}
printf("st f16/bf16 simd: tail-remainder slices ok\n");
free(all);
if (failures) { fprintf(stderr, "st f16/bf16 simd: %d failure(s)\n", failures); return 1; }
puts("st f16/bf16 simd: ok");
return 0;
}