def forward(self, model, sample, reduce=True, log_pred=False):
        """Compute the loss for the given sample.
        Returns a tuple with three elements:
        1) the loss
        2) the sample size, which is used as the denominator for the gradient
        3) logging outputs to display while training
        """
        reduction = "sum" if reduce else "none"

        if "net_input" in sample:
            text_sample = None
        else:
            text_sample = sample.get("text_paired")
            sample = sample.get("speech")

        ### 1. L_UMLM: do hubert forward and loss computation
        sample["modality"] = "speech"
        net_output = model(target_list=sample["target_list"], **sample["net_input"])
        loss, sample_size, logging_output = self.compute_hubert_loss(
            model,
            net_output,
            reduction,
        )
        if self.loss_weights is not None:
            assert hasattr(model, "get_extra_losses")
            extra_losses, names = model.get_extra_losses(net_output)
            if torch.is_tensor(extra_losses):
                extra_losses = [extra_losses]
                names = [names]
            if len(self.loss_weights) == 1 and len(extra_losses) != 1:
                self.loss_weights = [self.loss_weights[0]] * len(extra_losses)
            assert len(extra_losses) == len(
                self.loss_weights
            ), f"{len(extra_losses)}, {len(self.loss_weights)}"
            for p, n, coef in zip(extra_losses, names, self.loss_weights):
                if coef != 0 and p is not None:
                    p = coef * p.float() * sample_size
                    loss += p
                    logging_output[f"loss_{n}"] = p.item()
        for lk in self.log_keys:
            if lk in net_output:
                logging_output[lk] = float((net_output[lk]))
        
        ### 2. do text forward and loss computation
        if text_sample is not None:
            text_sample["modality"] = "text"
            ## 2.1 re-loading "target_list", in default case, target_list = [src_tokens],
            ## while in case of using "unit-phone-char" structure, target_list will be [ref_tokens]
            text_sample["net_input"]["target_list"] = [
                text_sample.get("ref_tokens", text_sample["net_input"]["src_tokens"].clone()),
            ]
            text_net_output = model(**text_sample["net_input"])

            ### 2.2 L_UMLM (text-end, not applied by default)
            if self.text_mum_weight > 0:
                loss_u2t, sample_size_u2t, logging_output_u2t = self.compute_hubert_loss(
                    model,
                    text_net_output,
                    reduction,
                    suffix="_u2t",
                )
                loss += self.text_mum_weight * loss_u2t * sample_size / sample_size_u2t
                logging_output.update(logging_output_u2t)

            ### 2.3 L_UCTC
            text_sample_size = text_sample["ntokens"]
            if self.text_ctc_weight > 0:
                text_ctc_loss = self.compute_ctc_loss(model, text_net_output, text_sample["target"], reduction=reduction)
                loss += self.text_ctc_weight * text_ctc_loss * sample_size / text_sample_size
                logging_output["text_ctc_loss"] = utils.item(text_ctc_loss)
                logging_output["text_sample_size"] = text_sample_size

        logging_output = {
            "loss": utils.item(loss) if reduce else loss,
            "ntokens": sample_size,
            "nsentences": sample["id"].numel() + (text_sample["id"].numel() if text_sample is not None else 0),
            "sample_size": sample_size,
            **logging_output,
        }

        return loss, sample_size, logging_output
Compute the loss for the given sample.
        Returns a tuple with three elements:
        1) the loss
        2) the sample size, which is used as the denominator for the gradient
        3) logging outputs to display while training
        
speechlm/speechlm/criterions/speechlm_criterion.py