def transformer_moe_8k_lm():
  """Language modeling params.

  Will have the following architecture by default:
  * No encoder.
  * Decoder architecture:
    * Layer 0: a - sepm  (masked self-attention/masked separable convolutions)
    * Layer 1: a - sepm
    * Layer 2: a - moe  (mixture of expert layers in the middle)
    * Layer 3: a - sepm
    * Layer 4: a - sepm

  Returns:
    hparams
  """
  hparams = transformer_moe_8k()

  # Use masked versions of local attention and separable convolution
  hparams.default_ff = "sepm"

  # hparams.layer_types contains the network architecture:
  # Start with '#' for decoder only architecture
  hparams.layer_types = "#a/a/a-moe/a/a"  # 5 full attention layers with 1 moe
  # For long sequences, if running out of memory, it's possible to use the
  # one of those two optimized versions instead:
  #  * Memory efficient multihead attention (slow):
  # hparams.layer_types = "#mem/mem/mem-moe/mem/mem"
  #  * Alternate between local/compressed attention layers (faster):
  # hparams.layer_types = "#locm/redm/locm-moe/redm/locm"

  return hparams
Language modeling params.

  Will have the following architecture by default:
  * No encoder.
  * Decoder architecture:
    * Layer 0: a - sepm  (masked self-attention/masked separable convolutions)
    * Layer 1: a - sepm
    * Layer 2: a - moe  (mixture of expert layers in the middle)
    * Layer 3: a - sepm
    * Layer 4: a - sepm

  Returns:
    hparams
  
tensor2tensor/models/research/transformer_moe.py