def multihead_attn(q, k, v, attn_mask, d_model, n_head, d_head, dropout,
                   dropatt, is_training, kernel_initializer, residual=True,
                   scope='abs_attn', reuse=None):
  """Standard multi-head attention with absolute positional embedding."""

  scale = 1 / (d_head ** 0.5)
  with tf.variable_scope(scope, reuse=reuse):
    # attention heads
    q_head = head_projection(
        q, d_model, n_head, d_head, kernel_initializer, 'q')
    k_head = head_projection(
        k, d_model, n_head, d_head, kernel_initializer, 'k')
    v_head = head_projection(
        v, d_model, n_head, d_head, kernel_initializer, 'v')

    # attention vector
    attn_vec = abs_attn_core(q_head, k_head, v_head, attn_mask, dropatt,
                             is_training, scale)

    # post processing
    output = post_attention(v, attn_vec, d_model, n_head, d_head, dropout,
                            is_training, kernel_initializer, residual)

  return output
Standard multi-head attention with absolute positional embedding.
src/modeling.py