Vocabulary projection
×learnedself.lm_head(hidden_states[:, slice_indices, :])
Project each normalized feature vector into vocabulary logits.
- in
- [B batch, T positions, 6144 d]
- out
- [B batch, T positions, 154880 vocabulary]
- parameters
- 951,582,720
Logits stay in the activations’ type, with no float32 cast (modeling_glm_moe_dsa.py 800–802).
Parameters
951,582,720 parameters per instance.
| Component | Calculation | Parameters |
|---|---|---|
| Weight matrix | 951,582,720 | |
| Per-instance total | 951,582,720 | |
The reference module holds the weight output by input, [out, in].
Derived from the configured matrix dimensions; the model total matches the checkpoint’s tensors.
Explanation
Each entry scores one vocabulary token as the next token for its position. The softmax and the choice of token happen outside the model.
When generating, logits_to_keep can limit it to the last positions (modeling_glm_moe_dsa.py 801–802).
Codemodeling_glm_moe_dsa.py 648–802 · config.json 1–59
_keys_to_ignore_on_load_unexpected = [r"model\.layers\.78.*"] if inputs_embeds is None:
inputs_embeds: torch.Tensor = self.embed_tokens(input_ids)
if use_cache and past_key_values is None:
past_key_values = DynamicCache(config=self.config)
if position_ids is None:
past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0
position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens
position_ids = position_ids.unsqueeze(0)
# It may already have been prepared by e.g. `generate`
if not isinstance(causal_mask_mapping := attention_mask, dict):
mask_kwargs = {
"config": self.config,
"inputs_embeds": inputs_embeds,
"attention_mask": attention_mask,
"past_key_values": past_key_values,
"position_ids": position_ids,
"allow_is_causal_skip": False, # Always force creation to account for causality in the indexer
}
causal_mask_mapping = {"indexed_attention": create_causal_mask(**mask_kwargs)}
hidden_states = inputs_embeds
position_embeddings = self.rotary_emb(hidden_states, position_ids=position_ids)
topk_indices = None # MAIN DIFF with DSV3.2
for i, decoder_layer in enumerate(self.layers[: self.config.num_hidden_layers]):
hidden_states, topk_indices = decoder_layer(
hidden_states,
attention_mask=causal_mask_mapping["indexed_attention"],
position_embeddings=position_embeddings,
position_ids=position_ids,
past_key_values=past_key_values,
use_cache=use_cache,
prev_topk_indices=topk_indices, # MAIN DIFF with DSV3.2
**kwargs,
)
hidden_states = self.norm(hidden_states)
return BaseModelOutputWithPast(
last_hidden_state=hidden_states,
past_key_values=past_key_values,
) def __init__(self, config):
super().__init__(config)
self.model = GlmMoeDsaModel(config)
self.vocab_size = config.vocab_size
self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) hidden_states = outputs.last_hidden_state
# Only compute necessary logits, and do not upcast them to float if we are not computing the loss
slice_indices = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep
logits = self.lm_head(hidden_states[:, slice_indices, :]){
"architectures": [
"GlmMoeDsaForCausalLM"
],
"attention_bias": false,
"attention_dropout": 0.0,
"dtype": "bfloat16",
"eos_token_id": [
154820,
154827,
154829
],
"ep_size": 1,
"first_k_dense_replace": 3,
"hidden_act": "silu",
"head_dim": 64,
"hidden_size": 6144,
"index_head_dim": 128,
"index_n_heads": 32,
"index_topk": 2048,
"indexer_rope_interleave": true,
"initializer_range": 0.02,
"intermediate_size": 12288,
"kv_lora_rank": 512,
"max_position_embeddings": 202752,
"moe_intermediate_size": 2048,
"moe_layer_freq": 1,
"model_type": "glm_moe_dsa",
"n_group": 1,
"n_routed_experts": 256,
"n_shared_experts": 1,
"norm_topk_prob": true,
"num_attention_heads": 64,
"num_experts_per_tok": 8,
"num_hidden_layers": 78,
"num_key_value_heads": 64,
"num_nextn_predict_layers": 1,
"pad_token_id": 154820,
"pretraining_tp": 1,
"q_lora_rank": 2048,
"qk_head_dim": 256,
"qk_nope_head_dim": 192,
"qk_rope_head_dim": 64,
"rms_norm_eps": 1e-05,
"rope_interleave": true,
"rope_parameters": {
"rope_theta": 1000000,
"rope_type": "default"
},
"routed_scaling_factor": 2.5,
"scoring_func": "sigmoid",
"tie_word_embeddings": false,
"topk_group": 1,
"topk_method": "noaux_tc",
"transformers_version": "5.0.2.dev0",
"use_cache": true,
"v_head_dim": 256,
"vocab_size": 154880
}Sources2
Direct evidence
The model embeds tokens, numbers positions from the cached length, always builds the causal mask (the indexer needs it), applies the 78 blocks, normalizes and projects to vocabulary logits with an untied lm_head without upcasting them. The checkpoint’s layer 78, the multi-token-prediction layer, is ignored on load.
transformer · huggingface/transformers / modeling_glm_moe_dsa.py
Pinned revision 5e4d6304de55 ↗ · lines 648–802
Read code excerpt
_keys_to_ignore_on_load_unexpected = [r"model\.layers\.78.*"]… lines 649–695 omitted …
if inputs_embeds is None:
inputs_embeds: torch.Tensor = self.embed_tokens(input_ids)
if use_cache and past_key_values is None:
past_key_values = DynamicCache(config=self.config)
if position_ids is None:
past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0
position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + past_seen_tokens
position_ids = position_ids.unsqueeze(0)
# It may already have been prepared by e.g. `generate`
if not isinstance(causal_mask_mapping := attention_mask, dict):
mask_kwargs = {
"config": self.config,
"inputs_embeds": inputs_embeds,
"attention_mask": attention_mask,
"past_key_values": past_key_values,
"position_ids": position_ids,
"allow_is_causal_skip": False, # Always force creation to account for causality in the indexer
}
causal_mask_mapping = {"indexed_attention": create_causal_mask(**mask_kwargs)}
hidden_states = inputs_embeds
position_embeddings = self.rotary_emb(hidden_states, position_ids=position_ids)
topk_indices = None # MAIN DIFF with DSV3.2
for i, decoder_layer in enumerate(self.layers[: self.config.num_hidden_layers]):
hidden_states, topk_indices = decoder_layer(
hidden_states,
attention_mask=causal_mask_mapping["indexed_attention"],
position_embeddings=position_embeddings,
position_ids=position_ids,
past_key_values=past_key_values,
use_cache=use_cache,
prev_topk_indices=topk_indices, # MAIN DIFF with DSV3.2
**kwargs,
)
hidden_states = self.norm(hidden_states)
return BaseModelOutputWithPast(
last_hidden_state=hidden_states,
past_key_values=past_key_values,
)… lines 740–748 omitted …
def __init__(self, config):
super().__init__(config)
self.model = GlmMoeDsaModel(config)
self.vocab_size = config.vocab_size
self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)… lines 754–798 omitted …
hidden_states = outputs.last_hidden_state
# Only compute necessary logits, and do not upcast them to float if we are not computing the loss
slice_indices = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep
logits = self.lm_head(hidden_states[:, slice_indices, :])Direct evidence
The released configuration: width, 78 layers of which 3 dense, 64 heads, 256 routed experts and one shared, routing (sigmoid scores, one expert group, top-8, route scale 2.5), MLA ranks and head widths, the indexer (32 heads of 128, top 2,048, interleaved RoPE), vocabulary, default RoPE with θ = 1,000,000, and one multi-token-prediction layer.
config · zai-org/GLM-5 / config.json
Pinned revision c183ef8c61fa ↗ · lines 1–59
Read code excerpt
{
"architectures": [
"GlmMoeDsaForCausalLM"
],
"attention_bias": false,
"attention_dropout": 0.0,
"dtype": "bfloat16",
"eos_token_id": [
154820,
154827,
154829
],
"ep_size": 1,
"first_k_dense_replace": 3,
"hidden_act": "silu",
"head_dim": 64,
"hidden_size": 6144,
"index_head_dim": 128,
"index_n_heads": 32,
"index_topk": 2048,
"indexer_rope_interleave": true,
"initializer_range": 0.02,
"intermediate_size": 12288,
"kv_lora_rank": 512,
"max_position_embeddings": 202752,
"moe_intermediate_size": 2048,
"moe_layer_freq": 1,
"model_type": "glm_moe_dsa",
"n_group": 1,
"n_routed_experts": 256,
"n_shared_experts": 1,
"norm_topk_prob": true,
"num_attention_heads": 64,
"num_experts_per_tok": 8,
"num_hidden_layers": 78,
"num_key_value_heads": 64,
"num_nextn_predict_layers": 1,
"pad_token_id": 154820,
"pretraining_tp": 1,
"q_lora_rank": 2048,
"qk_head_dim": 256,
"qk_nope_head_dim": 192,
"qk_rope_head_dim": 64,
"rms_norm_eps": 1e-05,
"rope_interleave": true,
"rope_parameters": {
"rope_theta": 1000000,
"rope_type": "default"
},
"routed_scaling_factor": 2.5,
"scoring_func": "sigmoid",
"tie_word_embeddings": false,
"topk_group": 1,
"topk_method": "noaux_tc",
"transformers_version": "5.0.2.dev0",
"use_cache": true,
"v_head_dim": 256,
"vocab_size": 154880
}