Transformer · Target token IDs

Target token IDs

supplied · no parametersfeatures["targets"]

The target sentence as indices into the same vocabulary; in training, the reference translation.

shape: $I^{tgt}$ [B batch, T target positions]

Codetransformer.py 44–73

  def model_fn_body(self, features):
    # Remove dropout if not training
    hparams = copy.copy(self._hparams)
    targets = features["targets"]
    inputs = features.get("inputs")
    target_space = features.get("target_space_id")

    inputs = common_layers.flatten4d3d(inputs)
    targets = common_layers.flatten4d3d(targets)

    (encoder_input, encoder_attention_bias, _) = (transformer_prepare_encoder(
        inputs, target_space, hparams))
    (decoder_input, decoder_self_attention_bias) = transformer_prepare_decoder(
        targets, hparams)

    def residual_fn(x, y):
      return common_layers.layer_norm(x + tf.nn.dropout(
          y, 1.0 - hparams.residual_dropout))

    # encoder_input = tf.squeeze(encoder_input, 2)
    # decoder_input = tf.squeeze(decoder_input, 2)
    encoder_input = tf.nn.dropout(encoder_input, 1.0 - hparams.residual_dropout)
    decoder_input = tf.nn.dropout(decoder_input, 1.0 - hparams.residual_dropout)
    encoder_output = transformer_encoder(encoder_input, residual_fn,
                                         encoder_attention_bias, hparams)

    decoder_output = transformer_decoder(
        decoder_input, encoder_output, residual_fn, decoder_self_attention_bias,
        encoder_attention_bias, hparams)
    decoder_output = tf.expand_dims(decoder_output, 2)

Sources1

Contextual evidence

The model prepares the encoder and decoder inputs, applies dropout to both, runs the encoder and then the decoder over the encoder output; every sublayer is combined by layer_norm(x + dropout(y)).

model-fn · tensorflow/tensor2tensor / tensor2tensor/models/transformer.py
Pinned revision 0c66117d4297 ↗ · lines 44–73

Read code excerpt

  def model_fn_body(self, features):
    # Remove dropout if not training
    hparams = copy.copy(self._hparams)
    targets = features["targets"]
    inputs = features.get("inputs")
    target_space = features.get("target_space_id")

    inputs = common_layers.flatten4d3d(inputs)
    targets = common_layers.flatten4d3d(targets)

    (encoder_input, encoder_attention_bias, _) = (transformer_prepare_encoder(
        inputs, target_space, hparams))
    (decoder_input, decoder_self_attention_bias) = transformer_prepare_decoder(
        targets, hparams)

    def residual_fn(x, y):
      return common_layers.layer_norm(x + tf.nn.dropout(
          y, 1.0 - hparams.residual_dropout))

    # encoder_input = tf.squeeze(encoder_input, 2)
    # decoder_input = tf.squeeze(decoder_input, 2)
    encoder_input = tf.nn.dropout(encoder_input, 1.0 - hparams.residual_dropout)
    decoder_input = tf.nn.dropout(decoder_input, 1.0 - hparams.residual_dropout)
    encoder_output = transformer_encoder(encoder_input, residual_fn,
                                         encoder_attention_bias, hparams)

    decoder_output = transformer_decoder(
        decoder_input, encoder_output, residual_fn, decoder_self_attention_bias,
        encoder_attention_bias, hparams)
    decoder_output = tf.expand_dims(decoder_output, 2)