src/transformers/models/distilbert/modeling_flax_distilbert.py/FlaxFFN/setup
class FlaxFFN:    def setup(self):
        self.dropout = nn.Dropout(rate=self.config.dropout)
        self.chunk_size_feed_forward = self.config.chunk_size_feed_forward
        self.seq_len_dim = 1
        self.lin1 = nn.Dense(
            self.config.hidden_dim,
            dtype=self.dtype,
            kernel_init=jax.nn.initializers.normal(stddev=self.config.initializer_range),
        )
        self.lin2 = nn.Dense(
            self.config.dim,
            dtype=self.dtype,
            kernel_init=jax.nn.initializers.normal(stddev=self.config.initializer_range),
        )
        assert self.config.activation in [
            "relu",
            "gelu",
        ], f"activation ({self.config.activation}) must be in ['relu', 'gelu']"
        self.activation = ACT2FN[self.config.activation]