93 layers › First layer › Kimi Delta Attention (KDA) › Key heads
Short convolution
×learnedself.k_conv1d(…) · 583–600
Mix each feature with its own last 3 positions, then apply SiLU.
- in
- [B batch, 12288 features, 4 kept inputs]
- out
- [B batch, 1 position, 12288 heads × d_h]
- parameters
- 49,152
Parameters
49,152 parameters owned by this component
| Component | Calculation | Parameters |
|---|---|---|
| weights, no bias | 49,152 | |
| Per-instance total | 49,152 | |
Across 1 independent layers: parameters.
Derived from the pinned configuration and implementation, with the per-head log-scale at the checkpoint’s 128 elements where the code creates 96; equal to the checkpoint’s own tensors. The quantisation scales are not counted.
A depthwise convolution: 4 weights for each of the 12,288 features, no bias.
Explanation
The kernel reads the current position and the ones before it, never later ones, so the convolution is causal.
Codeshort_conv.py 24–80 · modeling_kimi_linear.py 478–541
class ShortConvolution(nn.Conv1d):
"""Short convolution layer for efficient causal convolution operations.
This class implements a depthwise 1D convolution with causal padding,
designed for efficient sequence processing. It supports multiple backends (Triton/CUDA)
and optional activation functions.
Args:
hidden_size (int): Number of input/output channels (must be equal for depthwise conv)
kernel_size (int): Size of the convolution kernel
bias (bool, optional): Whether to include learnable bias. Defaults to False.
activation (Optional[str], optional): Activation function ('silu' or 'swish'). Defaults to 'silu'.
backend (Optional[str], optional): Backend implementation ('triton' or 'cuda'). Defaults to 'triton'.
device (Optional[torch.device], optional): Device to place the layer on. Defaults to None.
dtype (Optional[torch.dtype], optional): Data type for layer parameters. Defaults to None.
**kwargs: Additional keyword arguments (deprecated 'use_fast_conv1d' supported for compatibility)
Attributes:
hidden_size (int): Number of channels
activation (Optional[str]): Selected activation function
backend (str): Actual backend being used (may differ from input due to availability)
Note:
- Uses depthwise convolution (groups=hidden_size) for efficiency
- Applies causal padding (kernel_size-1) to ensure no future information leakage
- Falls back to Triton backend if CUDA backend is unavailable
"""
def __init__(
self,
hidden_size: int,
kernel_size: int,
bias: bool = False,
activation: str | None = 'silu',
backend: str | None = 'triton',
device: torch.device | None = None,
dtype: torch.dtype | None = None,
**kwargs,
):
super().__init__(
in_channels=hidden_size,
out_channels=hidden_size,
kernel_size=kernel_size,
groups=hidden_size,
bias=bias,
padding=kernel_size - 1,
device=device,
dtype=dtype,
)
self.hidden_size = hidden_size
self.activation = None
if activation is not None:
assert activation in ['silu', 'swish'], f"Activation `{activation}` not supported yet."
self.activation = activation
def __init__(self, config: KimiLinearConfig, layer_idx: int):
super().__init__()
self.config = config
self.mode = "chunk"
self.hidden_size = config.hidden_size
self.conv_size = config.linear_attn_config["short_conv_kernel_size"]
self.head_dim = config.linear_attn_config["head_dim"]
self.num_heads = config.linear_attn_config["num_heads"]
self.head_k_dim = self.head_dim
self.num_k_heads = self.num_heads
self.layer_idx = layer_idx
assert self.mode in [
'chunk', 'fused_recurrent'], f"Not supported mode `{self.mode}`."
projection_k_size = self.head_k_dim * self.num_k_heads
projection_size = self.head_dim * self.num_heads
self.q_proj = nn.Linear(
self.hidden_size, projection_k_size, bias=False)
self.k_proj = nn.Linear(
self.hidden_size, projection_k_size, bias=False)
self.v_proj = nn.Linear(self.hidden_size, projection_size, bias=False)
self.q_conv1d = ShortConvolution(
hidden_size=projection_k_size,
kernel_size=self.conv_size,
activation='silu',
)
self.k_conv1d = ShortConvolution(
hidden_size=projection_k_size,
kernel_size=self.conv_size,
activation='silu',
)
self.v_conv1d = ShortConvolution(
hidden_size=projection_size,
kernel_size=self.conv_size,
activation='silu',
)
self.A_log = torch.nn.Parameter(torch.log(torch.empty(
self.num_heads, dtype=torch.float32).uniform_(1, 16)))
self.f_a_proj = nn.Linear(self.hidden_size, self.head_dim, bias=False)
self.f_b_proj = nn.Linear(self.head_dim, projection_size, bias=False)
self.dt_bias = nn.Parameter(
torch.empty(projection_size, dtype=torch.float32))
self.b_proj = nn.Linear(self.hidden_size, self.num_heads, bias=False)
self.use_full_rank_gate = config.linear_attn_config.get("use_full_rank_gate", False)
self.gate_lower_bound = config.linear_attn_config.get("gate_lower_bound", None)
if self.use_full_rank_gate:
self.g_proj = nn.Linear(self.hidden_size, projection_size, bias=False)
else:
self.g_a_proj = nn.Linear(self.hidden_size, self.head_dim, bias=False)
self.g_b_proj = nn.Linear(self.head_dim, projection_size, bias=False)
self.o_norm = FusedRMSNormGated(
self.head_dim, eps=config.rms_norm_eps, activation='sigmoid')
self.o_proj = nn.Linear(projection_size, self.hidden_size, bias=False)Sources2
Direct evidence
The short convolution is a depthwise, causal 1D convolution without bias, followed by SiLU.
short-conv · fla-org/flash-linear-attention / fla/modules/conv/short_conv.py
Pinned revision 07ca1e49ef6c ↗ · lines 24–80
Read code excerpt
class ShortConvolution(nn.Conv1d):
"""Short convolution layer for efficient causal convolution operations.
This class implements a depthwise 1D convolution with causal padding,
designed for efficient sequence processing. It supports multiple backends (Triton/CUDA)
and optional activation functions.
Args:
hidden_size (int): Number of input/output channels (must be equal for depthwise conv)
kernel_size (int): Size of the convolution kernel
bias (bool, optional): Whether to include learnable bias. Defaults to False.
activation (Optional[str], optional): Activation function ('silu' or 'swish'). Defaults to 'silu'.
backend (Optional[str], optional): Backend implementation ('triton' or 'cuda'). Defaults to 'triton'.
device (Optional[torch.device], optional): Device to place the layer on. Defaults to None.
dtype (Optional[torch.dtype], optional): Data type for layer parameters. Defaults to None.
**kwargs: Additional keyword arguments (deprecated 'use_fast_conv1d' supported for compatibility)
Attributes:
hidden_size (int): Number of channels
activation (Optional[str]): Selected activation function
backend (str): Actual backend being used (may differ from input due to availability)
Note:
- Uses depthwise convolution (groups=hidden_size) for efficiency
- Applies causal padding (kernel_size-1) to ensure no future information leakage
- Falls back to Triton backend if CUDA backend is unavailable
"""
def __init__(
self,
hidden_size: int,
kernel_size: int,
bias: bool = False,
activation: str | None = 'silu',
backend: str | None = 'triton',
device: torch.device | None = None,
dtype: torch.dtype | None = None,
**kwargs,
):
super().__init__(
in_channels=hidden_size,
out_channels=hidden_size,
kernel_size=kernel_size,
groups=hidden_size,
bias=bias,
padding=kernel_size - 1,
device=device,
dtype=dtype,
)
self.hidden_size = hidden_size
self.activation = None
if activation is not None:
assert activation in ['silu', 'swish'], f"Activation `{activation}` not supported yet."
self.activation = activation
Direct evidence
Kimi Delta Attention owns query, key and value projections to 96 heads of 128 features, a causal short convolution on each, a per-head log-scale A_log, a rank-128 pair of projections and a bias for the decay logits, a per-head write-strength projection, a full-rank output gate projection, a gated RMSNorm over each head and an output projection.
kda-init · moonshotai/Kimi-K3 / modeling_kimi_linear.py
Pinned revision f831ab668142 ↗ · lines 478–541
Read code excerpt
def __init__(self, config: KimiLinearConfig, layer_idx: int):
super().__init__()
self.config = config
self.mode = "chunk"
self.hidden_size = config.hidden_size
self.conv_size = config.linear_attn_config["short_conv_kernel_size"]
self.head_dim = config.linear_attn_config["head_dim"]
self.num_heads = config.linear_attn_config["num_heads"]
self.head_k_dim = self.head_dim
self.num_k_heads = self.num_heads
self.layer_idx = layer_idx
assert self.mode in [
'chunk', 'fused_recurrent'], f"Not supported mode `{self.mode}`."
projection_k_size = self.head_k_dim * self.num_k_heads
projection_size = self.head_dim * self.num_heads
self.q_proj = nn.Linear(
self.hidden_size, projection_k_size, bias=False)
self.k_proj = nn.Linear(
self.hidden_size, projection_k_size, bias=False)
self.v_proj = nn.Linear(self.hidden_size, projection_size, bias=False)
self.q_conv1d = ShortConvolution(
hidden_size=projection_k_size,
kernel_size=self.conv_size,
activation='silu',
)
self.k_conv1d = ShortConvolution(
hidden_size=projection_k_size,
kernel_size=self.conv_size,
activation='silu',
)
self.v_conv1d = ShortConvolution(
hidden_size=projection_size,
kernel_size=self.conv_size,
activation='silu',
)
self.A_log = torch.nn.Parameter(torch.log(torch.empty(
self.num_heads, dtype=torch.float32).uniform_(1, 16)))
self.f_a_proj = nn.Linear(self.hidden_size, self.head_dim, bias=False)
self.f_b_proj = nn.Linear(self.head_dim, projection_size, bias=False)
self.dt_bias = nn.Parameter(
torch.empty(projection_size, dtype=torch.float32))
self.b_proj = nn.Linear(self.hidden_size, self.num_heads, bias=False)
self.use_full_rank_gate = config.linear_attn_config.get("use_full_rank_gate", False)
self.gate_lower_bound = config.linear_attn_config.get("gate_lower_bound", None)
if self.use_full_rank_gate:
self.g_proj = nn.Linear(self.hidden_size, projection_size, bias=False)
else:
self.g_a_proj = nn.Linear(self.hidden_size, self.head_dim, bias=False)
self.g_b_proj = nn.Linear(self.head_dim, projection_size, bias=False)
self.o_norm = FusedRMSNormGated(
self.head_dim, eps=config.rms_norm_eps, activation='sigmoid')
self.o_proj = nn.Linear(projection_size, self.hidden_size, bias=False)