def training_step(self, batch: tuple[torch.Tensor, torch.Tensor]):
        """Carries out a single update to actor and critic network from a batch of replay buffer.

        Args:
            batch: batch of replay buffer/trajectory data

        """
        state, action, old_logp, qval, adv = batch

        # normalize advantages
        adv = (adv - adv.mean()) / adv.std()

        self.log("avg_ep_len", self.avg_ep_len, prog_bar=True, on_step=False, on_epoch=True)
        self.log("avg_ep_reward", self.avg_ep_reward, prog_bar=True, on_step=False, on_epoch=True)
        self.log("avg_reward", self.avg_reward, prog_bar=True, on_step=False, on_epoch=True)

        optimizer_actor, optimizer_critic = self.optimizers()

        loss_actor = self.actor_loss(state, action, old_logp, qval, adv)
        self.manual_backward(loss_actor)
        optimizer_actor.step()
        optimizer_actor.zero_grad()

        loss_critic = self.critic_loss(state, action, old_logp, qval, adv)
        self.manual_backward(loss_critic)
        optimizer_critic.step()
        optimizer_critic.zero_grad()

        self.log("loss_critic", loss_critic, on_step=False, on_epoch=True, prog_bar=False, logger=True)
        self.log("loss_actor", loss_actor, on_step=False, on_epoch=True, prog_bar=True, logger=True)
Carries out a single update to actor and critic network from a batch of replay buffer.

        Args:
            batch: batch of replay buffer/trajectory data

        
examples/pytorch/domain_templates/reinforce_learn_ppo.py