def _optimize(self, obs, actions, rewards, obs_next, done):
        '''
        Note that while the replay contains uint8, the
        aggregator returns float32 tensors

        Arguments:
            obs: an observation from the minibatch, often represented as s_n in literature. Dimensionality: (N, C) for
                low dimensional inputs, (N, C, H, W) for pixel inputs
            actions: actions taken given observations obs, often represented as a_n in literature.
                Dimensionality: (N, A), where A is the dimensionality of a single action
            rewards: rewards received after action is taken. Dimensionality: N
            obs_next: an observation from the minibatch, often represented as s_{n+1} in literature
            done: 1 if obs_next is terminal, 0 otherwise. Dimensionality: N
        '''
        with tx.device_scope(self.gpu_ids):

            with self.forward_time.time():
                assert actions.max().item() <= 1.0
                assert actions.min().item() >= -1.0

                # estimate rewards using the next state: r + argmax_a Q'(s_{t+1}, u'(a))

                model_policy, next_Q_target = self.model_target.forward(obs_next)
                if self.use_action_regularization:
                    # https://github.com/sfujim/TD3/blob/master/TD3.py -- action regularization
                    policy_noise = 0.2
                    noise_clip = 0.5
                    batch_size = self.batch_size
                    noise = np.clip(np.random.normal(0, policy_noise, size=(batch_size, self.action_dim)), -noise_clip,
                                    noise_clip)
                    device_name = 'cpu'
                    if self._num_gpus > 0:
                        device_name = 'cuda'
                    model_policy += torch.tensor(noise, dtype=torch.float32).to(device_name).detach()
                    model_policy = model_policy.clamp(-1, 1).to(device_name)
                y = rewards + pow(self.discount_factor, self.n_step) * next_Q_target * (1.0 - done)
                if self.use_double_critic:
                    _, next_Q_target2 = self.model_target2.forward(obs_next, action=model_policy)
                    y2 = rewards + pow(self.discount_factor, self.n_step) * next_Q_target2 * (1.0 - done)
                    y = torch.min(y, y2)
                y = y.detach()

                # compute Q(s_t, a_t)
                perception = self.model.forward_perception(obs)
                y_policy = self.model.forward_critic(
                    perception,
                    actions.detach()
                )

                y_policy2 = None
                if self.use_double_critic:
                    perception2 = self.model2.forward_perception(obs)
                    y_policy2 = self.model2.forward_critic(
                        perception2,
                        actions.detach()
                    )

            # critic update
            with self.critic_update_time.time():
                self.model.critic.zero_grad()
                if self.is_pixel_input:
                    self.model.perception.zero_grad()
                critic_loss = self.critic_criterion(y_policy, y)        
                critic_loss.backward()
                if self.clip_critic_gradient:
                    self.model.critic.clip_grad_value(self.critic_gradient_clip_value)
                self.critic_optim.step()

                if self.use_double_critic:
                    self.model2.critic.zero_grad()
                    if self.is_pixel_input:
                        self.model2.perception.zero_grad()
                    critic_loss = self.critic_criterion(y_policy2, y)
                    critic_loss.backward()
                    if self.clip_critic_gradient:
                        self.model2.critic.clip_grad_value(self.critic_gradient_clip_value)
                    self.critic_optim2.step()

            # actor update
            with self.actor_update_time.time():
                self.model.actor.zero_grad()
                actor_loss = -self.model.forward_critic(
                    perception.detach(),
                    self.model.forward_actor(perception.detach())
                )
                actor_loss = actor_loss.mean()
                actor_loss.backward()
                if self.clip_actor_gradient:
                    self.model.actor.clip_grad_value(self.actor_gradient_clip_value)
                self.actor_optim.step()

            tensorplex_update_dict = {
                'actor_loss': actor_loss.item(),
                'critic_loss': critic_loss.item(),
                'action_norm': actions.norm(2, 1).mean().item(),
                'rewards': rewards.mean().item(),
                'Q_target': y.mean().item(),
                'Q_policy': y_policy.mean().item(),
                'performance/forward_time': self.forward_time.avg,
                'performance/critic_update_time': self.critic_update_time.avg,
                'performance/actor_update_time': self.actor_update_time.avg,
            }
            if self.use_double_critic:
                tensorplex_update_dict['Q_policy2'] = y_policy2.mean().item()

            # (possibly) update target networks
            self._target_update()

            return tensorplex_update_dict

        Note that while the replay contains uint8, the
        aggregator returns float32 tensors

        Arguments:
            obs: an observation from the minibatch, often represented as s_n in literature. Dimensionality: (N, C) for
                low dimensional inputs, (N, C, H, W) for pixel inputs
            actions: actions taken given observations obs, often represented as a_n in literature.
                Dimensionality: (N, A), where A is the dimensionality of a single action
            rewards: rewards received after action is taken. Dimensionality: N
            obs_next: an observation from the minibatch, often represented as s_{n+1} in literature
            done: 1 if obs_next is terminal, 0 otherwise. Dimensionality: N
        
surreal/learner/ddpg.py