- in
- Source token IDs
- out
- [B, T, 37000]
- parameters
- 63,098,880 in all
Parameters
63,098,880 parameters in Transformer
| Component | Calculation | Parameters |
|---|---|---|
| Token embedding | 18,944,000 | |
| Add target-space embedding | 16,384 | |
| Encoder: 6 layers | 18,914,304 | |
| Decoder: 6 layers | 25,224,192 | |
| Model total | 63,098,880 | |
Derived from the modules the pinned tensor2tensor v1.0.14 code creates, with a vocabulary of about 37,000 (the paper’s words); no trained weights were published to count.
Sources
Direct evidence
The model prepares the encoder and decoder inputs, applies dropout to both, runs the encoder and then the decoder over the encoder output; every sublayer is combined by layer_norm(x + dropout(y)).
model-fn · tensorflow/tensor2tensor / tensor2tensor/models/transformer.py
Pinned revision 0c66117d4297 ↗ · lines 44–73
Read code excerpt
def model_fn_body(self, features):
# Remove dropout if not training
hparams = copy.copy(self._hparams)
targets = features["targets"]
inputs = features.get("inputs")
target_space = features.get("target_space_id")
inputs = common_layers.flatten4d3d(inputs)
targets = common_layers.flatten4d3d(targets)
(encoder_input, encoder_attention_bias, _) = (transformer_prepare_encoder(
inputs, target_space, hparams))
(decoder_input, decoder_self_attention_bias) = transformer_prepare_decoder(
targets, hparams)
def residual_fn(x, y):
return common_layers.layer_norm(x + tf.nn.dropout(
y, 1.0 - hparams.residual_dropout))
# encoder_input = tf.squeeze(encoder_input, 2)
# decoder_input = tf.squeeze(decoder_input, 2)
encoder_input = tf.nn.dropout(encoder_input, 1.0 - hparams.residual_dropout)
decoder_input = tf.nn.dropout(decoder_input, 1.0 - hparams.residual_dropout)
encoder_output = transformer_encoder(encoder_input, residual_fn,
encoder_attention_bias, hparams)
decoder_output = transformer_decoder(
decoder_input, encoder_output, residual_fn, decoder_self_attention_bias,
encoder_attention_bias, hparams)
decoder_output = tf.expand_dims(decoder_output, 2)Direct evidence
transformer_base: width 512, 6 layers, 8 heads, filter size 2048, label smoothing 0.1, shared embedding and softmax weights, residual dropout 0.1, sinusoidal timing.
hparams-base · tensorflow/tensor2tensor / tensor2tensor/models/transformer.py
Pinned revision 0c66117d4297 ↗ · lines 269–306
Read code excerpt
def transformer_base():
"""Set of hyperparameters."""
hparams = common_hparams.basic_params1()
hparams.hidden_size = 512
hparams.batch_size = 4096
hparams.max_length = 256
hparams.dropout = 0.0
hparams.clip_grad_norm = 0. # i.e. no gradient clipping
hparams.optimizer_adam_epsilon = 1e-9
hparams.learning_rate_decay_scheme = "noam"
hparams.learning_rate = 0.1
hparams.learning_rate_warmup_steps = 4000
hparams.initializer_gain = 1.0
hparams.num_hidden_layers = 6
hparams.initializer = "uniform_unit_scaling"
hparams.weight_decay = 0.0
hparams.optimizer_adam_beta1 = 0.9
hparams.optimizer_adam_beta2 = 0.98
hparams.num_sampled_classes = 0
hparams.label_smoothing = 0.1
hparams.shared_embedding_and_softmax_weights = int(True)
hparams.add_hparam("filter_size", 2048) # Add new ones like this.
# attention-related flags
hparams.add_hparam("num_heads", 8)
hparams.add_hparam("attention_key_channels", 0)
hparams.add_hparam("attention_value_channels", 0)
hparams.add_hparam("ffn_layer", "conv_hidden_relu")
hparams.add_hparam("parameter_attention_key_channels", 0)
hparams.add_hparam("parameter_attention_value_channels", 0)
# All hyperparameters ending in "dropout" are automatically set to 0.0
# when not in training mode.
hparams.add_hparam("attention_dropout", 0.0)
hparams.add_hparam("relu_dropout", 0.0)
hparams.add_hparam("residual_dropout", 0.1)
hparams.add_hparam("pos", "timing") # timing, none
hparams.add_hparam("nbr_decoder_problems", 1)
return hparams