- in
- [B batch, T positions, 4 streams, 5120 d] [B batch, T positions, 4 streams] [B batch, T positions, 2 Engram layers, 24 n-grams × heads]
- out
- [B batch, T positions, 4 streams, 5120 d] [B batch, T positions, 4 streams]
- parameters
- 747,170,852,464
Parameters
747,170,852,464 parameters per instance, across its learned components
| Component | Calculation | Parameters |
|---|---|---|
| Window layer | 13,754,511,990 across its components | 13,754,511,990 |
| Engram | 98,462,906,368 across its components | 98,462,906,368 |
| Window layer | 13,754,511,990 across its components | 13,754,511,990 |
| Pooled group | 165,075,575,432 | |
| Engram | 98,465,597,952 across its components | 98,465,597,952 |
| Pooled group | 82,537,787,716 across its components | 82,537,787,716 |
| Token group | 55,026,142,296 across its components | 55,026,142,296 |
| Re-indexing group | 220,093,818,720 | |
| Per-instance total | 747,170,852,464 | |
Derived from the pinned reference implementation and configuration; equal to the element counts of the checkpoint’s own tensors on the text path. The quantisation scales, the vision encoder and the DSpark blocks are not counted.
Explanation
A source layer compresses the whole sequence so far into one 512-feature vector per 2 tokens (layers 2, 8, 14) or per token (layer 20), and chooses 512 of them for every query. The layers after it borrow both until the next source.
From layer 24 on, every 4th layer chooses again with its own queries, against layer 20’s keys and inside layer 20’s 2,048 candidate blocks; the keys and values stay layer 20’s.
Codemodel.py 613–1272 · config.json 1–66
class Transformer(nn.Module):
"""DeepSeek-V4.1: embed -> expand to hc_mult copies -> blocks -> collapse -> logits. Building
this sets the globals at the top of the file. The tokenizer only feeds the engram token map."""
def __init__(self, args: ModelArgs, tokenizer=None):
global world_size, rank, default_dtype
world_size = dist.get_world_size() if dist.is_initialized() else 1
rank = dist.get_rank() if dist.is_initialized() else 0
default_dtype = torch.float8_e4m3fn if args.dtype == "fp8" else torch.bfloat16
super().__init__()
self.max_seq_len = args.max_seq_len
self.temperature = args.temperature
self.norm_eps = args.norm_eps
self.hc_eps = args.hc_eps
self.engram_layout = EngramLayout.from_args(args)
self.engram_hash = (
NgramHashState(args, self.engram_layout, tokenizer) if self.engram_layout is not None else None
)
self.embed = ParallelEmbedding(args.vocab_size, args.dim)
self.layers = torch.nn.ModuleList()
for layer_id in range(args.n_layers):
self.layers.append(Block(layer_id, args, self.engram_layout))
self.norm = RMSNorm(args.dim, self.norm_eps)
self.head = ParallelHead(args.vocab_size, args.dim, self.norm_eps, self.hc_eps)
self.mtp = torch.nn.ModuleList()
self.target_layer_ids = args.dspark_target_layer_ids
if args.dspark_block_size:
for layer_id in range(args.n_mtp_layers):
self.mtp.append(DSparkBlock(args.n_layers + layer_id, args))
self.mtp[-1].embed = self.embed
self.mtp[-1].head = self.head
self.hc_mult = args.hc_mult
self.vision = None
if args.vision_enabled:
self.vision = ViT(args)
self.aligner = Aligner(args)
# learned embeddings for the image span delimiters
self.image_start = nn.Parameter(torch.empty(args.dim))
self.image_end = nn.Parameter(torch.empty(args.dim))
self.image_newline = nn.Parameter(torch.empty(args.dim))
@torch.inference_mode()
def forward(
self, input_ids: torch.Tensor, start_pos: int = 0, images=None, token_types: torch.Tensor | None = None
):
"""input_ids: [b, s], every entry a real token id -- generate.py only ever passes positions it
has already filled, so the padding it uses internally never reaches here. token_types /
images carry the VL inputs built by image_processor.prepare_vl_inputs; image spans must lie
inside the first (start_pos 0) chunk."""
image_mask = None if token_types is None else token_types >= 0 # TEXT is -1
# image tokens take no part in an n-gram and get no engram contribution; text-only needs no mask
engram_mask = None if image_mask is None else ~image_mask
engram_hashes = self.engram_hash(input_ids, start_pos, engram_mask) if self.engram_hash is not None else None
h = self.embed(input_ids)
if images is not None:
assert start_pos == 0, "image spans must be prefilled in a single chunk"
self.merge_image_embeddings(images, h)
# Expand to hc_mult copies for Hyper-Connections
h = h.unsqueeze(2).repeat(1, 1, self.hc_mult, 1)
main_hiddens = []
pre_mix = make_identity_pre_mix(h, self.hc_mult)
for i, layer in enumerate(self.layers):
if layer.engram is not None:
h = layer.engram(h, engram_hashes[:, :, layer.engram.layer_hash_index, :], engram_mask)
# the MTP head reads the attention input of its target layers, not their output
if i in self.target_layer_ids:
main_hiddens.append(h.mean(dim=2))
h, pre_mix = layer(h, start_pos, pre_mix, image_mask)
h = layer.hc_pre(h, pre_mix)
logits = self.head(self.norm(h))
output_ids = sample(logits, self.temperature)
main_hidden = torch.cat(main_hiddens, dim=-1) if main_hiddens else None
return output_ids, logits, main_hiddenclass Attention(nn.Module):
"""Latent attention over two KV sources at once, concatenated into one `sparse_attn` call: a
sliding window of raw KV, plus -- when compress_ratio > 0 -- `index_topk` compressed positions
reaching further back. Q and the output projection are both low-rank, the latter grouped.
compress_ratio > 0 does not mean the layer compresses its own KV: only kv_source_layers do,
the rest read that same cache.
"""
def __init__(self, layer_id: int, args: ModelArgs):
super().__init__()
self.layer_id = layer_id
self.dim = args.dim
self.n_heads = args.n_heads
self.n_local_heads = args.n_heads // world_size
self.q_lora_rank = args.q_lora_rank
self.o_lora_rank = args.o_lora_rank
self.head_dim = args.head_dim
self.rope_head_dim = args.rope_head_dim
self.nope_head_dim = args.head_dim - args.rope_head_dim
self.n_groups = args.o_groups
self.n_local_groups = self.n_groups // world_size
self.window_size = args.window_size
self.compress_ratio = args.compress_ratios[layer_id]
self.eps = args.norm_eps
self.attn_sink = nn.Parameter(torch.empty(self.n_local_heads, dtype=torch.float32))
self.wq_a = Linear(self.dim, self.q_lora_rank)
self.q_norm = RMSNorm(self.q_lora_rank, self.eps)
self.wq_b = ColumnParallelLinear(self.q_lora_rank, self.n_heads * self.head_dim)
self.wkv = Linear(self.dim, self.head_dim)
self.kv_norm = RMSNorm(self.head_dim, self.eps)
self.wo_a = ColumnParallelLinear(
self.n_heads * self.head_dim // self.n_groups,
self.n_groups * args.o_lora_rank,
dtype=torch.bfloat16,
)
self.wo_b = RowParallelLinear(self.n_groups * args.o_lora_rank, self.dim)
self.softmax_scale = self.head_dim**-0.5
is_backbone = layer_id < args.n_layers
self.is_kv_source = is_backbone and layer_id in args.kv_source_layers
self.is_index_source = is_backbone and layer_id in args.index_source_layers
self.compressor: Compressor | None = None
self.indexer: Indexer | None = None
if self.is_kv_source:
self.compressor = Compressor(args, layer_id)
if self.is_index_source:
self.indexer = Indexer(args, layer_id)
self.window_kv_cache: torch.Tensor
self.register_buffer(
"window_kv_cache",
torch.zeros(args.max_batch_size, args.window_size, self.head_dim),
persistent=False,
)
if self.is_kv_source:
self.compress_kv_cache: torch.Tensor
self.register_buffer(
"compress_kv_cache",
torch.zeros(
args.max_batch_size,
args.max_seq_len // self.compress_ratio,
self.head_dim,
),
persistent=False,
)
if self.compress_ratio:
original_seq_len, rope_theta = (
args.original_seq_len,
args.compress_rope_theta,
)
else:
# disable YaRN and use base rope_theta in pure sliding-window attention
original_seq_len, rope_theta = 0, args.rope_theta
freqs_cis = precompute_freqs_cis(
self.rope_head_dim,
args.max_seq_len,
original_seq_len,
rope_theta,
args.rope_factor,
args.beta_fast,
args.beta_slow,
)
self.freqs_cis: torch.Tensor
self.register_buffer("freqs_cis", freqs_cis, persistent=False)class SharedAttentionRuntime:
"""What attention layers hand down the stack instead of recomputing. Layers run in order and
every source writes before its consumers read, so one slot each is enough and nothing needs
resetting between forwards. Sources: compress_kv and index_k from kv_source_layers, topk_idxs
from index_source_layers, candidates from candidate_source_layer."""
def __init__(self):
self.compress_kv: torch.Tensor | None = None
self.index_k: torch.Tensor | None = None
self.topk_idxs: torch.Tensor | None = None
self.candidates: torch.Tensor | None = None
# Only ever one model per process, same as world_size / rank / default_dtype above.
shared_attn = SharedAttentionRuntime(){
"vocab_size": 129280,
"dim": 5120,
"moe_inter_dim": 2304,
"n_layers": 40,
"n_mtp_layers": 3,
"dspark_block_size": 5,
"dspark_noise_token_id": 128799,
"dspark_target_layer_ids": [37, 38, 39],
"dspark_markov_rank": 256,
"dspark_n_routed_experts": 128,
"dspark_n_activated_experts": 3,
"n_heads": 64,
"n_routed_experts": 384,
"n_shared_experts": 1,
"n_activated_experts": 6,
"score_func": "sqrtsoftplus",
"route_scale": 1.5,
"swiglu_limit": 10.0,
"q_lora_rank": 1280,
"head_dim": 512,
"rope_head_dim": 64,
"norm_eps": 1e-20,
"o_groups": 8,
"o_lora_rank": 1024,
"window_size": 128,
"kv_source_layers": [2, 8, 14, 20],
"index_source_layers": [2, 8, 14, 20, 24, 28, 32, 36],
"original_seq_len": 65536,
"rope_theta": 10000,
"rope_factor": 16,
"beta_fast": 32,
"beta_slow": 1,
"index_n_heads": 32,
"index_head_dim": 128,
"index_topk": 512,
"candidate_source_layer": 20,
"candidate_topk_blocks": 2048,
"candidate_block_size": 8,
"hc_mult": 4,
"hc_sinkhorn_iters": 20,
"hc_eps": 1e-06,
"engram_layer_ids": [1, 14],
"engram_vocab_size": 16000000,
"engram_num_embeddings": [384006168, 384016682],
"engram_max_ngram_size": 4,
"engram_pad_id": 2,
"engram_compressed_vocab_size": 99092,
"dtype": "fp8",
"expert_dtype": "fp4",
"compress_rope_theta": 160000,
"compress_ratios": [0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0],
"vision_n_layers": 32,
"vision_dim": 1024,
"vision_n_heads": 16,
"vision_inter_dim": 2816,
"vision_patch_size": 14,
"vision_downsample_ratio": 3,
"vision_max_n_token": 1024,
"vision_min_pixels": 295936,
"vision_max_wh_ratio": null,
"image_token_id": 129264,
"engram_n_heads": 8,
"engram_head_dim": 256,
"vision_rope_theta": 10000
}Sources5
Direct evidence
The model owns the embedding, 40 blocks, the final norm and the head, with the n-gram hashing and, when configured, a vision encoder.
transformer-init · deepseek-ai/DeepSeek-V4.1-Flash / inference/model.py
Pinned revision 2cba9e42aa02 ↗ · lines 1183–1223
Read code excerpt
class Transformer(nn.Module):
"""DeepSeek-V4.1: embed -> expand to hc_mult copies -> blocks -> collapse -> logits. Building
this sets the globals at the top of the file. The tokenizer only feeds the engram token map."""
def __init__(self, args: ModelArgs, tokenizer=None):
global world_size, rank, default_dtype
world_size = dist.get_world_size() if dist.is_initialized() else 1
rank = dist.get_rank() if dist.is_initialized() else 0
default_dtype = torch.float8_e4m3fn if args.dtype == "fp8" else torch.bfloat16
super().__init__()
self.max_seq_len = args.max_seq_len
self.temperature = args.temperature
self.norm_eps = args.norm_eps
self.hc_eps = args.hc_eps
self.engram_layout = EngramLayout.from_args(args)
self.engram_hash = (
NgramHashState(args, self.engram_layout, tokenizer) if self.engram_layout is not None else None
)
self.embed = ParallelEmbedding(args.vocab_size, args.dim)
self.layers = torch.nn.ModuleList()
for layer_id in range(args.n_layers):
self.layers.append(Block(layer_id, args, self.engram_layout))
self.norm = RMSNorm(args.dim, self.norm_eps)
self.head = ParallelHead(args.vocab_size, args.dim, self.norm_eps, self.hc_eps)
self.mtp = torch.nn.ModuleList()
self.target_layer_ids = args.dspark_target_layer_ids
if args.dspark_block_size:
for layer_id in range(args.n_mtp_layers):
self.mtp.append(DSparkBlock(args.n_layers + layer_id, args))
self.mtp[-1].embed = self.embed
self.mtp[-1].head = self.head
self.hc_mult = args.hc_mult
self.vision = None
if args.vision_enabled:
self.vision = ViT(args)
self.aligner = Aligner(args)
# learned embeddings for the image span delimiters
self.image_start = nn.Parameter(torch.empty(args.dim))
self.image_end = nn.Parameter(torch.empty(args.dim))
self.image_newline = nn.Parameter(torch.empty(args.dim))
Direct evidence
Tokens are embedded and repeated into four streams; each layer, after an Engram step where it has one, reads and writes the streams; the last reading mix collapses them, then the final norm and the head.
transformer-forward · deepseek-ai/DeepSeek-V4.1-Flash / inference/model.py
Pinned revision 2cba9e42aa02 ↗ · lines 1241–1272
Read code excerpt
@torch.inference_mode()
def forward(
self, input_ids: torch.Tensor, start_pos: int = 0, images=None, token_types: torch.Tensor | None = None
):
"""input_ids: [b, s], every entry a real token id -- generate.py only ever passes positions it
has already filled, so the padding it uses internally never reaches here. token_types /
images carry the VL inputs built by image_processor.prepare_vl_inputs; image spans must lie
inside the first (start_pos 0) chunk."""
image_mask = None if token_types is None else token_types >= 0 # TEXT is -1
# image tokens take no part in an n-gram and get no engram contribution; text-only needs no mask
engram_mask = None if image_mask is None else ~image_mask
engram_hashes = self.engram_hash(input_ids, start_pos, engram_mask) if self.engram_hash is not None else None
h = self.embed(input_ids)
if images is not None:
assert start_pos == 0, "image spans must be prefilled in a single chunk"
self.merge_image_embeddings(images, h)
# Expand to hc_mult copies for Hyper-Connections
h = h.unsqueeze(2).repeat(1, 1, self.hc_mult, 1)
main_hiddens = []
pre_mix = make_identity_pre_mix(h, self.hc_mult)
for i, layer in enumerate(self.layers):
if layer.engram is not None:
h = layer.engram(h, engram_hashes[:, :, layer.engram.layer_hash_index, :], engram_mask)
# the MTP head reads the attention input of its target layers, not their output
if i in self.target_layer_ids:
main_hiddens.append(h.mean(dim=2))
h, pre_mix = layer(h, start_pos, pre_mix, image_mask)
h = layer.hc_pre(h, pre_mix)
logits = self.head(self.norm(h))
output_ids = sample(logits, self.temperature)
main_hidden = torch.cat(main_hiddens, dim=-1) if main_hiddens else None
return output_ids, logits, main_hiddenDirect evidence
Attention owns a low-rank query, one key–value projection, a learned sink per head, a grouped low-rank output, a ring buffer for the window, and, by the two source lists, a compressor and an indexer; layers that attend to compressed positions use the larger rotary base with YaRN.
attention-init · deepseek-ai/DeepSeek-V4.1-Flash / inference/model.py
Pinned revision 2cba9e42aa02 ↗ · lines 613–698
Read code excerpt
class Attention(nn.Module):
"""Latent attention over two KV sources at once, concatenated into one `sparse_attn` call: a
sliding window of raw KV, plus -- when compress_ratio > 0 -- `index_topk` compressed positions
reaching further back. Q and the output projection are both low-rank, the latter grouped.
compress_ratio > 0 does not mean the layer compresses its own KV: only kv_source_layers do,
the rest read that same cache.
"""
def __init__(self, layer_id: int, args: ModelArgs):
super().__init__()
self.layer_id = layer_id
self.dim = args.dim
self.n_heads = args.n_heads
self.n_local_heads = args.n_heads // world_size
self.q_lora_rank = args.q_lora_rank
self.o_lora_rank = args.o_lora_rank
self.head_dim = args.head_dim
self.rope_head_dim = args.rope_head_dim
self.nope_head_dim = args.head_dim - args.rope_head_dim
self.n_groups = args.o_groups
self.n_local_groups = self.n_groups // world_size
self.window_size = args.window_size
self.compress_ratio = args.compress_ratios[layer_id]
self.eps = args.norm_eps
self.attn_sink = nn.Parameter(torch.empty(self.n_local_heads, dtype=torch.float32))
self.wq_a = Linear(self.dim, self.q_lora_rank)
self.q_norm = RMSNorm(self.q_lora_rank, self.eps)
self.wq_b = ColumnParallelLinear(self.q_lora_rank, self.n_heads * self.head_dim)
self.wkv = Linear(self.dim, self.head_dim)
self.kv_norm = RMSNorm(self.head_dim, self.eps)
self.wo_a = ColumnParallelLinear(
self.n_heads * self.head_dim // self.n_groups,
self.n_groups * args.o_lora_rank,
dtype=torch.bfloat16,
)
self.wo_b = RowParallelLinear(self.n_groups * args.o_lora_rank, self.dim)
self.softmax_scale = self.head_dim**-0.5
is_backbone = layer_id < args.n_layers
self.is_kv_source = is_backbone and layer_id in args.kv_source_layers
self.is_index_source = is_backbone and layer_id in args.index_source_layers
self.compressor: Compressor | None = None
self.indexer: Indexer | None = None
if self.is_kv_source:
self.compressor = Compressor(args, layer_id)
if self.is_index_source:
self.indexer = Indexer(args, layer_id)
self.window_kv_cache: torch.Tensor
self.register_buffer(
"window_kv_cache",
torch.zeros(args.max_batch_size, args.window_size, self.head_dim),
persistent=False,
)
if self.is_kv_source:
self.compress_kv_cache: torch.Tensor
self.register_buffer(
"compress_kv_cache",
torch.zeros(
args.max_batch_size,
args.max_seq_len // self.compress_ratio,
self.head_dim,
),
persistent=False,
)
if self.compress_ratio:
original_seq_len, rope_theta = (
args.original_seq_len,
args.compress_rope_theta,
)
else:
# disable YaRN and use base rope_theta in pure sliding-window attention
original_seq_len, rope_theta = 0, args.rope_theta
freqs_cis = precompute_freqs_cis(
self.rope_head_dim,
args.max_seq_len,
original_seq_len,
rope_theta,
args.rope_factor,
args.beta_fast,
args.beta_slow,
)
self.freqs_cis: torch.Tensor
self.register_buffer("freqs_cis", freqs_cis, persistent=False)Direct evidence
One object hands compressed keys and values, index keys, selected positions and candidates from a source layer to the layers after it.
shared · deepseek-ai/DeepSeek-V4.1-Flash / inference/model.py
Pinned revision 2cba9e42aa02 ↗ · lines 1166–1180
Read code excerpt
class SharedAttentionRuntime:
"""What attention layers hand down the stack instead of recomputing. Layers run in order and
every source writes before its consumers read, so one slot each is enough and nothing needs
resetting between forwards. Sources: compress_kv and index_k from kv_source_layers, topk_idxs
from index_source_layers, candidates from candidate_source_layer."""
def __init__(self):
self.compress_kv: torch.Tensor | None = None
self.index_k: torch.Tensor | None = None
self.topk_idxs: torch.Tensor | None = None
self.candidates: torch.Tensor | None = None
# Only ever one model per process, same as world_size / rank / default_dtype above.
shared_attn = SharedAttentionRuntime()Direct evidence
The reference configuration: 40 layers of width 5,120, 64 heads of 512 features with a 64-feature rotary part, a 1,280-feature query latent, a grouped output of 8 groups at rank 1,024, a window of 128, the per-layer compression ratios and the two source lists, the indexer and its candidate stage, hyper-connections of 4 streams, 384 routed experts of which 6 are used with 1 shared, Engram before layers 1 and 14, and three DSpark blocks.
config · deepseek-ai/DeepSeek-V4.1-Flash / inference/config.json
Pinned revision 2cba9e42aa02 ↗ · lines 1–66
Read code excerpt
{
"vocab_size": 129280,
"dim": 5120,
"moe_inter_dim": 2304,
"n_layers": 40,
"n_mtp_layers": 3,
"dspark_block_size": 5,
"dspark_noise_token_id": 128799,
"dspark_target_layer_ids": [37, 38, 39],
"dspark_markov_rank": 256,
"dspark_n_routed_experts": 128,
"dspark_n_activated_experts": 3,
"n_heads": 64,
"n_routed_experts": 384,
"n_shared_experts": 1,
"n_activated_experts": 6,
"score_func": "sqrtsoftplus",
"route_scale": 1.5,
"swiglu_limit": 10.0,
"q_lora_rank": 1280,
"head_dim": 512,
"rope_head_dim": 64,
"norm_eps": 1e-20,
"o_groups": 8,
"o_lora_rank": 1024,
"window_size": 128,
"kv_source_layers": [2, 8, 14, 20],
"index_source_layers": [2, 8, 14, 20, 24, 28, 32, 36],
"original_seq_len": 65536,
"rope_theta": 10000,
"rope_factor": 16,
"beta_fast": 32,
"beta_slow": 1,
"index_n_heads": 32,
"index_head_dim": 128,
"index_topk": 512,
"candidate_source_layer": 20,
"candidate_topk_blocks": 2048,
"candidate_block_size": 8,
"hc_mult": 4,
"hc_sinkhorn_iters": 20,
"hc_eps": 1e-06,
"engram_layer_ids": [1, 14],
"engram_vocab_size": 16000000,
"engram_num_embeddings": [384006168, 384016682],
"engram_max_ngram_size": 4,
"engram_pad_id": 2,
"engram_compressed_vocab_size": 99092,
"dtype": "fp8",
"expert_dtype": "fp4",
"compress_rope_theta": 160000,
"compress_ratios": [0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0],
"vision_n_layers": 32,
"vision_dim": 1024,
"vision_n_heads": 16,
"vision_inter_dim": 2816,
"vision_patch_size": 14,
"vision_downsample_ratio": 3,
"vision_max_n_token": 1024,
"vision_min_pixels": 295936,
"vision_max_wh_ratio": null,
"image_token_id": 129264,
"engram_n_heads": 8,
"engram_head_dim": 256,
"vision_rope_theta": 10000
}