40 layers
learnedself.layers
Re-indexing layer, then 3 layers that reuse its choice of positions.
All 34 steps
- in
- [B batch, T positions, 4 streams, 5120 d] [B batch, T positions, 4 streams] [B batch, T compressed positions, 512 d_h] [B batch, T compressed positions, 128 index width] [B batch, T positions, T compressed positions]
- out
- [B batch, T positions, 4 streams, 5120 d]
- parameters
- 55,023,454,680 · × 4 groups = 220,093,818,720
Between forward calls, in each of the 4 groups: each of the 4 layers keeps a ring of 128 slots holding the window keys and values of up to the last 128 positions (model.py 663–668, 708–719). These layers read compressed rows kept by a source layer before them; re-indexing layers also read index keys and candidates and compute their own selected positions, while reusing layers read the published selected positions. They allocate none of those shared buffers (model.py 654–661, 669, 500, 517, 724–737, 763, 1166–1180).
Parameters
55,023,454,680 parameters per group, across its learned components
| Component | Calculation | Parameters |
|---|---|---|
| Re-indexing layer | 13,759,918,710 across its components | 13,759,918,710 |
| Reusing layer | 41,263,535,970 | |
| Per-instance total | 55,023,454,680 | |
Across 4 independent groups: parameters.
Derived from the pinned reference implementation and configuration; equal to the element counts of the checkpoint’s own tensors on the text path. The quantisation scales, the vision encoder and the DSpark blocks are not counted.
Explanation
4 such groups: layers 24–27, layers 28–31, layers 32–35, layers 36–39. The layers that borrow recompute nothing: they read one shared object the source wrote (model.py 1166–1180).
Codemodel.py 722–763, 1166–1180 · config.json 1–66
class SharedAttentionRuntime:
"""What attention layers hand down the stack instead of recomputing. Layers run in order and
every source writes before its consumers read, so one slot each is enough and nothing needs
resetting between forwards. Sources: compress_kv and index_k from kv_source_layers, topk_idxs
from index_source_layers, candidates from candidate_source_layer."""
def __init__(self):
self.compress_kv: torch.Tensor | None = None
self.index_k: torch.Tensor | None = None
self.topk_idxs: torch.Tensor | None = None
self.candidates: torch.Tensor | None = None
# Only ever one model per process, same as world_size / rank / default_dtype above.
shared_attn = SharedAttentionRuntime() def _compress_topk_idxs(self, x, qr, latent, start_pos, offset, compress_len):
"""Which compressed positions each query attends to. Index sources run their own indexer;
the layers in between reuse the result their source published."""
if not self.is_index_source:
return shared_attn.topk_idxs
bsz, seqlen, _ = x.size()
if compress_len == 0:
idxs = torch.empty(bsz, seqlen, 0, dtype=torch.int32, device=x.device)
else:
assert self.indexer is not None
if self.indexer.freqs_cis is None:
self.indexer.freqs_cis = self.freqs_cis
idxs = self.indexer(x, qr, latent, start_pos, offset)
shared_attn.topk_idxs = idxs
return idxs
def _compress_kv(self, x, qr, start_pos, offset):
"""The shared compressed KV and the compressed positions every query may attend to. This
layer compresses its own KV only when it is a source; otherwise it just reads the cache."""
bsz, seqlen, _ = x.size()
ratio = self.compress_ratio
compress_len = (start_pos + seqlen) // ratio
latent = None
if self.is_kv_source:
latent = self.compressor(x, start_pos)
shared_attn.compress_kv = self.compress_kv_cache
# the indexer needs the latent before RoPE, so it runs before the cache is written
idxs = self._compress_topk_idxs(x, qr, latent, start_pos, offset, compress_len)
if latent is not None:
# a latent stands for the first token of its group, so group j takes position j * ratio
freqs = (
self.freqs_cis[: seqlen - seqlen % ratio : ratio]
if start_pos == 0
else self.freqs_cis[start_pos + 1 - ratio].unsqueeze(0)
)
apply_rotary_emb(latent[..., -self.rope_head_dim :], freqs)
# Compressed KV uses groups of 16 with E4M3 scales; the indexer uses 32 with E8M0.
fp4_act_quant(latent, 16, True, scale_dtype=torch.float8_e4m3fn)
self.compress_kv_cache[:bsz, start_pos // ratio : start_pos // ratio + latent.size(1)] = latent
# read after the write, so this does not depend on the slice aliasing the cache
return shared_attn.compress_kv[:bsz, :compress_len], idxs{
"vocab_size": 129280,
"dim": 5120,
"moe_inter_dim": 2304,
"n_layers": 40,
"n_mtp_layers": 3,
"dspark_block_size": 5,
"dspark_noise_token_id": 128799,
"dspark_target_layer_ids": [37, 38, 39],
"dspark_markov_rank": 256,
"dspark_n_routed_experts": 128,
"dspark_n_activated_experts": 3,
"n_heads": 64,
"n_routed_experts": 384,
"n_shared_experts": 1,
"n_activated_experts": 6,
"score_func": "sqrtsoftplus",
"route_scale": 1.5,
"swiglu_limit": 10.0,
"q_lora_rank": 1280,
"head_dim": 512,
"rope_head_dim": 64,
"norm_eps": 1e-20,
"o_groups": 8,
"o_lora_rank": 1024,
"window_size": 128,
"kv_source_layers": [2, 8, 14, 20],
"index_source_layers": [2, 8, 14, 20, 24, 28, 32, 36],
"original_seq_len": 65536,
"rope_theta": 10000,
"rope_factor": 16,
"beta_fast": 32,
"beta_slow": 1,
"index_n_heads": 32,
"index_head_dim": 128,
"index_topk": 512,
"candidate_source_layer": 20,
"candidate_topk_blocks": 2048,
"candidate_block_size": 8,
"hc_mult": 4,
"hc_sinkhorn_iters": 20,
"hc_eps": 1e-06,
"engram_layer_ids": [1, 14],
"engram_vocab_size": 16000000,
"engram_num_embeddings": [384006168, 384016682],
"engram_max_ngram_size": 4,
"engram_pad_id": 2,
"engram_compressed_vocab_size": 99092,
"dtype": "fp8",
"expert_dtype": "fp4",
"compress_rope_theta": 160000,
"compress_ratios": [0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0],
"vision_n_layers": 32,
"vision_dim": 1024,
"vision_n_heads": 16,
"vision_inter_dim": 2816,
"vision_patch_size": 14,
"vision_downsample_ratio": 3,
"vision_max_n_token": 1024,
"vision_min_pixels": 295936,
"vision_max_wh_ratio": null,
"image_token_id": 129264,
"engram_n_heads": 8,
"engram_head_dim": 256,
"vision_rope_theta": 10000
}Sources3
Direct evidence
One object hands compressed keys and values, index keys, selected positions and candidates from a source layer to the layers after it.
shared · deepseek-ai/DeepSeek-V4.1-Flash / inference/model.py
Pinned revision 2cba9e42aa02 ↗ · lines 1166–1180
Read code excerpt
class SharedAttentionRuntime:
"""What attention layers hand down the stack instead of recomputing. Layers run in order and
every source writes before its consumers read, so one slot each is enough and nothing needs
resetting between forwards. Sources: compress_kv and index_k from kv_source_layers, topk_idxs
from index_source_layers, candidates from candidate_source_layer."""
def __init__(self):
self.compress_kv: torch.Tensor | None = None
self.index_k: torch.Tensor | None = None
self.topk_idxs: torch.Tensor | None = None
self.candidates: torch.Tensor | None = None
# Only ever one model per process, same as world_size / rank / default_dtype above.
shared_attn = SharedAttentionRuntime()Direct evidence
A key–value source compresses its own input and writes the shared cache; an index source runs its indexer before the latent is rotated; every other layer reads what the nearest earlier source published.
attention-compress · deepseek-ai/DeepSeek-V4.1-Flash / inference/model.py
Pinned revision 2cba9e42aa02 ↗ · lines 722–763
Read code excerpt
def _compress_topk_idxs(self, x, qr, latent, start_pos, offset, compress_len):
"""Which compressed positions each query attends to. Index sources run their own indexer;
the layers in between reuse the result their source published."""
if not self.is_index_source:
return shared_attn.topk_idxs
bsz, seqlen, _ = x.size()
if compress_len == 0:
idxs = torch.empty(bsz, seqlen, 0, dtype=torch.int32, device=x.device)
else:
assert self.indexer is not None
if self.indexer.freqs_cis is None:
self.indexer.freqs_cis = self.freqs_cis
idxs = self.indexer(x, qr, latent, start_pos, offset)
shared_attn.topk_idxs = idxs
return idxs
def _compress_kv(self, x, qr, start_pos, offset):
"""The shared compressed KV and the compressed positions every query may attend to. This
layer compresses its own KV only when it is a source; otherwise it just reads the cache."""
bsz, seqlen, _ = x.size()
ratio = self.compress_ratio
compress_len = (start_pos + seqlen) // ratio
latent = None
if self.is_kv_source:
latent = self.compressor(x, start_pos)
shared_attn.compress_kv = self.compress_kv_cache
# the indexer needs the latent before RoPE, so it runs before the cache is written
idxs = self._compress_topk_idxs(x, qr, latent, start_pos, offset, compress_len)
if latent is not None:
# a latent stands for the first token of its group, so group j takes position j * ratio
freqs = (
self.freqs_cis[: seqlen - seqlen % ratio : ratio]
if start_pos == 0
else self.freqs_cis[start_pos + 1 - ratio].unsqueeze(0)
)
apply_rotary_emb(latent[..., -self.rope_head_dim :], freqs)
# Compressed KV uses groups of 16 with E4M3 scales; the indexer uses 32 with E8M0.
fp4_act_quant(latent, 16, True, scale_dtype=torch.float8_e4m3fn)
self.compress_kv_cache[:bsz, start_pos // ratio : start_pos // ratio + latent.size(1)] = latent
# read after the write, so this does not depend on the slice aliasing the cache
return shared_attn.compress_kv[:bsz, :compress_len], idxsDirect evidence
The reference configuration: 40 layers of width 5,120, 64 heads of 512 features with a 64-feature rotary part, a 1,280-feature query latent, a grouped output of 8 groups at rank 1,024, a window of 128, the per-layer compression ratios and the two source lists, the indexer and its candidate stage, hyper-connections of 4 streams, 384 routed experts of which 6 are used with 1 shared, Engram before layers 1 and 14, and three DSpark blocks.
config · deepseek-ai/DeepSeek-V4.1-Flash / inference/config.json
Pinned revision 2cba9e42aa02 ↗ · lines 1–66
Read code excerpt
{
"vocab_size": 129280,
"dim": 5120,
"moe_inter_dim": 2304,
"n_layers": 40,
"n_mtp_layers": 3,
"dspark_block_size": 5,
"dspark_noise_token_id": 128799,
"dspark_target_layer_ids": [37, 38, 39],
"dspark_markov_rank": 256,
"dspark_n_routed_experts": 128,
"dspark_n_activated_experts": 3,
"n_heads": 64,
"n_routed_experts": 384,
"n_shared_experts": 1,
"n_activated_experts": 6,
"score_func": "sqrtsoftplus",
"route_scale": 1.5,
"swiglu_limit": 10.0,
"q_lora_rank": 1280,
"head_dim": 512,
"rope_head_dim": 64,
"norm_eps": 1e-20,
"o_groups": 8,
"o_lora_rank": 1024,
"window_size": 128,
"kv_source_layers": [2, 8, 14, 20],
"index_source_layers": [2, 8, 14, 20, 24, 28, 32, 36],
"original_seq_len": 65536,
"rope_theta": 10000,
"rope_factor": 16,
"beta_fast": 32,
"beta_slow": 1,
"index_n_heads": 32,
"index_head_dim": 128,
"index_topk": 512,
"candidate_source_layer": 20,
"candidate_topk_blocks": 2048,
"candidate_block_size": 8,
"hc_mult": 4,
"hc_sinkhorn_iters": 20,
"hc_eps": 1e-06,
"engram_layer_ids": [1, 14],
"engram_vocab_size": 16000000,
"engram_num_embeddings": [384006168, 384016682],
"engram_max_ngram_size": 4,
"engram_pad_id": 2,
"engram_compressed_vocab_size": 99092,
"dtype": "fp8",
"expert_dtype": "fp4",
"compress_rope_theta": 160000,
"compress_ratios": [0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0],
"vision_n_layers": 32,
"vision_dim": 1024,
"vision_n_heads": 16,
"vision_inter_dim": 2816,
"vision_patch_size": 14,
"vision_downsample_ratio": 3,
"vision_max_n_token": 1024,
"vision_min_pixels": 295936,
"vision_max_wh_ratio": null,
"image_token_id": 129264,
"engram_n_heads": 8,
"engram_head_dim": 256,
"vision_rope_theta": 10000
}