40 layers › 6 layers › 5 reusing layers › Stream weights for attention
Stream combination
×learnedcomb · kernel.py 430–458
A 4 × 4 matrix that carries the streams over, with rows and columns that each sum close to 1.
- in
- [B batch, T positions, 24 mix numbers]
- out
- [B batch, T positions, 4 streams, 4 streams]
- parameters
- 17 · × 5 layers = 85
After the row softmax: one column normalization, then 19 rounds of a row and a column normalization, each dividing by the sum plus ϵ = 0.000001 (kernel.py 445–458); the row softmax itself has ϵ added to every entry (kernel.py 443). Streams j and k count from 1. 16 offsets and one scale.
Parameters
17 parameters owned by this component
| Component | Calculation | Parameters |
|---|---|---|
| learned vector | 16 | |
| learned vector | 1 | |
| Per-instance total | 17 | |
Across 5 independent layers: parameters.
Derived from the pinned reference implementation and configuration; equal to the element counts of the checkpoint’s own tensors on the text path. The quantisation scales, the vision encoder and the DSpark blocks are not counted.
Explanation
A matrix whose rows and columns all sum to 1 redistributes the streams without scaling them as a whole.
Codekernel.py 406–474 · config.json 1–66
@tilelang.jit(pass_configs=pass_configs)
def hc_split_sinkhorn_kernel(hc: int, sinkhorn_iters: int, eps: float):
n = T.symbolic("n")
mix_hc = (2 + hc) * hc
threads = 64
@T.prim_func
def hc_split_sinkhorn_kernel_(
mixes: T.Tensor[(n, mix_hc), FP32],
hc_scale: T.Tensor[(3,), FP32],
hc_base: T.Tensor[(mix_hc,), FP32],
pre: T.Tensor[(n, hc), FP32],
post: T.Tensor[(n, hc), FP32],
comb: T.Tensor[(n, hc, hc), FP32],
):
with T.Kernel(n, threads=threads) as i:
mixes_shared = T.alloc_shared(mix_hc, FP32)
comb_frag = T.alloc_fragment((hc, hc), FP32)
T.copy(mixes[i, :], mixes_shared)
for j in T.Parallel(hc):
pre[i, j] = T.sigmoid(mixes_shared[j] * hc_scale[0] + hc_base[j]) + eps
for j in T.Parallel(hc):
post[i, j] = 2 * T.sigmoid(mixes_shared[j + hc] * hc_scale[1] + hc_base[j + hc])
for j, k in T.Parallel(hc, hc):
comb_frag[j, k] = mixes_shared[j * hc + k + hc * 2] * hc_scale[2] + hc_base[j * hc + k + hc * 2]
row_sum = T.alloc_fragment(hc, FP32)
col_sum = T.alloc_fragment(hc, FP32)
# comb = comb.softmax(-1) + eps
row_max = T.alloc_fragment(hc, FP32)
T.reduce_max(comb_frag, row_max, dim=1)
for j, k in T.Parallel(hc, hc):
comb_frag[j, k] = T.exp(comb_frag[j, k] - row_max[j])
T.reduce_sum(comb_frag, row_sum, dim=1)
for j, k in T.Parallel(hc, hc):
comb_frag[j, k] = comb_frag[j, k] / row_sum[j] + eps
# comb = comb / (comb.sum(-2) + eps)
T.reduce_sum(comb_frag, col_sum, dim=0)
for j, k in T.Parallel(hc, hc):
comb_frag[j, k] = comb_frag[j, k] / (col_sum[k] + eps)
for _ in T.serial(sinkhorn_iters - 1):
# comb = comb / (comb.sum(-1) + eps)
T.reduce_sum(comb_frag, row_sum, dim=1)
for j, k in T.Parallel(hc, hc):
comb_frag[j, k] = comb_frag[j, k] / (row_sum[j] + eps)
# comb = comb / (comb.sum(-2) + eps)
T.reduce_sum(comb_frag, col_sum, dim=0)
for j, k in T.Parallel(hc, hc):
comb_frag[j, k] = comb_frag[j, k] / (col_sum[k] + eps)
T.copy(comb_frag, comb[i, :, :])
return hc_split_sinkhorn_kernel_
def hc_split_sinkhorn(
mixes: torch.Tensor, hc_scale: torch.Tensor, hc_base: torch.Tensor, hc_mult: int = 4, sinkhorn_iters: int = 20, eps: float = 1e-6
):
b, s, _ = mixes.size()
pre = mixes.new_empty(b, s, hc_mult)
post = mixes.new_empty(b, s, hc_mult)
comb = mixes.new_empty(b, s, hc_mult, hc_mult)
kernel = hc_split_sinkhorn_kernel(hc_mult, sinkhorn_iters, eps)
kernel(mixes.view(-1, (2 + hc_mult) * hc_mult), hc_scale, hc_base, pre.view(-1, hc_mult), post.view(-1, hc_mult), comb.view(-1, hc_mult, hc_mult))
return pre, post, comb{
"vocab_size": 129280,
"dim": 5120,
"moe_inter_dim": 2304,
"n_layers": 40,
"n_mtp_layers": 3,
"dspark_block_size": 5,
"dspark_noise_token_id": 128799,
"dspark_target_layer_ids": [37, 38, 39],
"dspark_markov_rank": 256,
"dspark_n_routed_experts": 128,
"dspark_n_activated_experts": 3,
"n_heads": 64,
"n_routed_experts": 384,
"n_shared_experts": 1,
"n_activated_experts": 6,
"score_func": "sqrtsoftplus",
"route_scale": 1.5,
"swiglu_limit": 10.0,
"q_lora_rank": 1280,
"head_dim": 512,
"rope_head_dim": 64,
"norm_eps": 1e-20,
"o_groups": 8,
"o_lora_rank": 1024,
"window_size": 128,
"kv_source_layers": [2, 8, 14, 20],
"index_source_layers": [2, 8, 14, 20, 24, 28, 32, 36],
"original_seq_len": 65536,
"rope_theta": 10000,
"rope_factor": 16,
"beta_fast": 32,
"beta_slow": 1,
"index_n_heads": 32,
"index_head_dim": 128,
"index_topk": 512,
"candidate_source_layer": 20,
"candidate_topk_blocks": 2048,
"candidate_block_size": 8,
"hc_mult": 4,
"hc_sinkhorn_iters": 20,
"hc_eps": 1e-06,
"engram_layer_ids": [1, 14],
"engram_vocab_size": 16000000,
"engram_num_embeddings": [384006168, 384016682],
"engram_max_ngram_size": 4,
"engram_pad_id": 2,
"engram_compressed_vocab_size": 99092,
"dtype": "fp8",
"expert_dtype": "fp4",
"compress_rope_theta": 160000,
"compress_ratios": [0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0],
"vision_n_layers": 32,
"vision_dim": 1024,
"vision_n_heads": 16,
"vision_inter_dim": 2816,
"vision_patch_size": 14,
"vision_downsample_ratio": 3,
"vision_max_n_token": 1024,
"vision_min_pixels": 295936,
"vision_max_wh_ratio": null,
"image_token_id": 129264,
"engram_n_heads": 8,
"engram_head_dim": 256,
"vision_rope_theta": 10000
}Sources2
Direct evidence
The 24 mix numbers become four reading weights (a sigmoid), four writing weights (twice a sigmoid) and a 4 × 4 combination made close to doubly stochastic: a softmax over rows, then alternating column and row normalisations.
hc-split · deepseek-ai/DeepSeek-V4.1-Flash / inference/kernel.py
Pinned revision 2cba9e42aa02 ↗ · lines 406–474
Read code excerpt
@tilelang.jit(pass_configs=pass_configs)
def hc_split_sinkhorn_kernel(hc: int, sinkhorn_iters: int, eps: float):
n = T.symbolic("n")
mix_hc = (2 + hc) * hc
threads = 64
@T.prim_func
def hc_split_sinkhorn_kernel_(
mixes: T.Tensor[(n, mix_hc), FP32],
hc_scale: T.Tensor[(3,), FP32],
hc_base: T.Tensor[(mix_hc,), FP32],
pre: T.Tensor[(n, hc), FP32],
post: T.Tensor[(n, hc), FP32],
comb: T.Tensor[(n, hc, hc), FP32],
):
with T.Kernel(n, threads=threads) as i:
mixes_shared = T.alloc_shared(mix_hc, FP32)
comb_frag = T.alloc_fragment((hc, hc), FP32)
T.copy(mixes[i, :], mixes_shared)
for j in T.Parallel(hc):
pre[i, j] = T.sigmoid(mixes_shared[j] * hc_scale[0] + hc_base[j]) + eps
for j in T.Parallel(hc):
post[i, j] = 2 * T.sigmoid(mixes_shared[j + hc] * hc_scale[1] + hc_base[j + hc])
for j, k in T.Parallel(hc, hc):
comb_frag[j, k] = mixes_shared[j * hc + k + hc * 2] * hc_scale[2] + hc_base[j * hc + k + hc * 2]
row_sum = T.alloc_fragment(hc, FP32)
col_sum = T.alloc_fragment(hc, FP32)
# comb = comb.softmax(-1) + eps
row_max = T.alloc_fragment(hc, FP32)
T.reduce_max(comb_frag, row_max, dim=1)
for j, k in T.Parallel(hc, hc):
comb_frag[j, k] = T.exp(comb_frag[j, k] - row_max[j])
T.reduce_sum(comb_frag, row_sum, dim=1)
for j, k in T.Parallel(hc, hc):
comb_frag[j, k] = comb_frag[j, k] / row_sum[j] + eps
# comb = comb / (comb.sum(-2) + eps)
T.reduce_sum(comb_frag, col_sum, dim=0)
for j, k in T.Parallel(hc, hc):
comb_frag[j, k] = comb_frag[j, k] / (col_sum[k] + eps)
for _ in T.serial(sinkhorn_iters - 1):
# comb = comb / (comb.sum(-1) + eps)
T.reduce_sum(comb_frag, row_sum, dim=1)
for j, k in T.Parallel(hc, hc):
comb_frag[j, k] = comb_frag[j, k] / (row_sum[j] + eps)
# comb = comb / (comb.sum(-2) + eps)
T.reduce_sum(comb_frag, col_sum, dim=0)
for j, k in T.Parallel(hc, hc):
comb_frag[j, k] = comb_frag[j, k] / (col_sum[k] + eps)
T.copy(comb_frag, comb[i, :, :])
return hc_split_sinkhorn_kernel_
def hc_split_sinkhorn(
mixes: torch.Tensor, hc_scale: torch.Tensor, hc_base: torch.Tensor, hc_mult: int = 4, sinkhorn_iters: int = 20, eps: float = 1e-6
):
b, s, _ = mixes.size()
pre = mixes.new_empty(b, s, hc_mult)
post = mixes.new_empty(b, s, hc_mult)
comb = mixes.new_empty(b, s, hc_mult, hc_mult)
kernel = hc_split_sinkhorn_kernel(hc_mult, sinkhorn_iters, eps)
kernel(mixes.view(-1, (2 + hc_mult) * hc_mult), hc_scale, hc_base, pre.view(-1, hc_mult), post.view(-1, hc_mult), comb.view(-1, hc_mult, hc_mult))
return pre, post, combDirect evidence
The reference configuration: 40 layers of width 5,120, 64 heads of 512 features with a 64-feature rotary part, a 1,280-feature query latent, a grouped output of 8 groups at rank 1,024, a window of 128, the per-layer compression ratios and the two source lists, the indexer and its candidate stage, hyper-connections of 4 streams, 384 routed experts of which 6 are used with 1 shared, Engram before layers 1 and 14, and three DSpark blocks.
config · deepseek-ai/DeepSeek-V4.1-Flash / inference/config.json
Pinned revision 2cba9e42aa02 ↗ · lines 1–66
Read code excerpt
{
"vocab_size": 129280,
"dim": 5120,
"moe_inter_dim": 2304,
"n_layers": 40,
"n_mtp_layers": 3,
"dspark_block_size": 5,
"dspark_noise_token_id": 128799,
"dspark_target_layer_ids": [37, 38, 39],
"dspark_markov_rank": 256,
"dspark_n_routed_experts": 128,
"dspark_n_activated_experts": 3,
"n_heads": 64,
"n_routed_experts": 384,
"n_shared_experts": 1,
"n_activated_experts": 6,
"score_func": "sqrtsoftplus",
"route_scale": 1.5,
"swiglu_limit": 10.0,
"q_lora_rank": 1280,
"head_dim": 512,
"rope_head_dim": 64,
"norm_eps": 1e-20,
"o_groups": 8,
"o_lora_rank": 1024,
"window_size": 128,
"kv_source_layers": [2, 8, 14, 20],
"index_source_layers": [2, 8, 14, 20, 24, 28, 32, 36],
"original_seq_len": 65536,
"rope_theta": 10000,
"rope_factor": 16,
"beta_fast": 32,
"beta_slow": 1,
"index_n_heads": 32,
"index_head_dim": 128,
"index_topk": 512,
"candidate_source_layer": 20,
"candidate_topk_blocks": 2048,
"candidate_block_size": 8,
"hc_mult": 4,
"hc_sinkhorn_iters": 20,
"hc_eps": 1e-06,
"engram_layer_ids": [1, 14],
"engram_vocab_size": 16000000,
"engram_num_embeddings": [384006168, 384016682],
"engram_max_ngram_size": 4,
"engram_pad_id": 2,
"engram_compressed_vocab_size": 99092,
"dtype": "fp8",
"expert_dtype": "fp4",
"compress_rope_theta": 160000,
"compress_ratios": [0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0],
"vision_n_layers": 32,
"vision_dim": 1024,
"vision_n_heads": 16,
"vision_inter_dim": 2816,
"vision_patch_size": 14,
"vision_downsample_ratio": 3,
"vision_max_n_token": 1024,
"vision_min_pixels": 295936,
"vision_max_wh_ratio": null,
"image_token_id": 129264,
"engram_n_heads": 8,
"engram_head_dim": 256,
"vision_rope_theta": 10000
}