def _ground_truth_calculation(discounts, log_rhos, rewards, q_values, values,
                              bootstrap_value, lambda_):
  """Calculates the ground truth for Retrace in python/numpy."""
  qs = []
  seq_len = len(discounts)
  rhos = np.exp(log_rhos)
  cs = np.minimum(rhos, 1.0)
  cs *= lambda_
  # This is a very inefficient way to calculate the Retrace ground truth.
  values_t_plus_1 = np.concatenate([values, bootstrap_value[None, :]], axis=0)
  for s in range(seq_len):
    q_s = np.copy(q_values[s])  # Very important copy...
    delta = rewards[s] + discounts[s] * values_t_plus_1[s + 1] - q_values[s]
    q_s += delta
    for t in range(s + 1, seq_len):
      q_s += (
          np.prod(discounts[s:t], axis=0) * np.prod(cs[s + 1:t + 1], axis=0) *
          (rewards[t] + discounts[t] * values_t_plus_1[t + 1] - q_values[t]))
    qs.append(q_s)
  qs = np.stack(qs, axis=0)
  return qs
Calculates the ground truth for Retrace in python/numpy.
trfl/continuous_retrace_ops_test.py