src/transformers/models/switch_transformers/modeling_switch_transformers.py/SwitchTransformersBlock/__init__
class SwitchTransformersBlock:    def __init__(self, config, has_relative_attention_bias=False, is_sparse=False):
        super().__init__()
        self.is_decoder = config.is_decoder
        self.is_sparse = is_sparse
        self.layer = nn.ModuleList()
        self.layer.append(
            SwitchTransformersLayerSelfAttention(config, has_relative_attention_bias=has_relative_attention_bias)
        )
        if self.is_decoder:
            self.layer.append(SwitchTransformersLayerCrossAttention(config))

        self.layer.append(SwitchTransformersLayerFF(config, is_sparse=self.is_sparse))