src/transformers/models/xglm/modeling_tf_xglm.py/create_sinusiodal_positions
def create_sinusiodal_positions(num_positions: int, embedding_dim: int, padding_idx: Optional[int]) -> tf.Tensor:
    half_dim = embedding_dim // 2
    emb = math.log(10000) / (half_dim - 1)
    emb = tf.exp(tf.range(half_dim, dtype=tf.float32) * -emb)
    emb = tf.expand_dims(tf.range(num_positions, dtype=tf.float32), axis=1) * tf.expand_dims(emb, axis=0)
    emb = tf.reshape(tf.concat([tf.sin(emb), tf.cos(emb)], axis=1), (num_positions, -1))
    if embedding_dim % 2 == 1:
        # zero pad
        emb = tf.concat([emb, tf.zeros((num_positions, 1))], axis=1)
    if padding_idx is not None:
        _padding_mask = tf.concat(
            [
                tf.ones((padding_idx, shape_list(emb)[1])),
                tf.zeros((1, shape_list(emb)[1])),
                tf.ones((shape_list(emb)[0] - padding_idx - 1, shape_list(emb)[1])),
            ],
            axis=0,
        )
        emb *= _padding_mask

    return tf.Variable(emb, trainable=False, name="model.embed_positions.weights")