def forward(self, input_):
        """Forward of RowParallelLinear

        Args:
            input_: 3D tensor whose order of dimension is [sequence, batch, hidden]

        Returns:
            - output
            - bias
        """
        # Set up backprop all-reduce.
        if self.input_is_parallel:
            input_parallel = input_
        else:
            assert not self.sequence_parallel
            input_parallel = ScatterToModelParallelRegion.apply(input_)
        # Matrix multiply.
        self._forward_impl = linear_with_grad_accumulation_and_async_allreduce
        output_parallel = self._forward_impl(
            input=input_parallel,
            weight=self.weight,
            bias=None,
            gradient_accumulation_fusion=self.gradient_accumulation_fusion,
            async_grad_allreduce=False,
            sequence_parallel=False,
        )

        # All-reduce across all the partitions.
        # if self.sequence_parallel:
        #     output_ = reduce_scatter_to_sequence_parallel_region(output_parallel)
        # else:
        #     output_ = reduce_from_tensor_model_parallel_region(output_parallel)
        # if not self.skip_bias_add:
        #     output = output_ + self.bias if self.bias is not None else output_
        #     output_bias = None
        # else:
        #     output = output_
        #     output_bias = self.bias

        output = output_parallel
        output_bias = self.bias
        return output, output_bias
Forward of RowParallelLinear

        Args:
            input_: 3D tensor whose order of dimension is [sequence, batch, hidden]

        Returns:
            - output
            - bias
        
training/DeepSpeed-Domino/domino/tensor_parallel/partition.py