def train_keep_hidden(self, sample_buffer, print_every=500):
        """ add new samples in sample_buffer to replay buffer and train
            keep hidden state (split episode into small sequence, but keep hidden states)
            this means must train some episodes continuously not fully random.
            to use this training scheme, you should also modify self._calc_target

        Parameters
        ----------
        sample_buffer: magent.utility.EpisodesBuffer
            buffer contains samples
        print_every: int
            print log every print_every batches

        Returns
        -------
        loss: float
            bellman residual loss
        value: float
            estimated state value
        """

        add_num = self._add_to_replay_buffer(sample_buffer)
   
        batch_size = self.batch_size
        unroll_step = self.unroll_step

        # calc sample weight of episodes (i.e. their lengths)
        replay_buffer = self.replay_buffer
        replay_lens_sum = np.sum(self.replay_buffer_lens)
        weight = np.array(self.replay_buffer_lens, dtype=np.float32) / replay_lens_sum
   
        max_len  = self._div_round(np.max(self.replay_buffer_lens), unroll_step)
        n_batches = self.train_freq * add_num / (batch_size * unroll_step)
        if n_batches == 0:
            return 0, 0

        # allocate buffer
        max_ = batch_size * max_len
        batch_view     = np.zeros((max_+1,) + self.view_space, dtype=np.float32)
        batch_feature  = np.zeros((max_+1,) + self.feature_space, dtype=np.float32)
        batch_action   = np.zeros((max_,), dtype=np.int32)
        batch_reward   = np.zeros((max_,), dtype=np.float32)
        batch_terminal = np.zeros((max_,), dtype=np.bool)
        batch_mask     = np.zeros((max_,), dtype=np.float32)
        batch_hidden   = np.zeros((batch_size, self.state_size), dtype=np.float32)
        batch_pick     = np.zeros((batch_size, max_len), dtype=np.bool)
        pick_buffer    = np.arange(max_, dtype=np.int32)

        print("batches: %d  add: %d  replay_len: %d, %d/%d" %
              (n_batches, add_num, replay_lens_sum, len(self.replay_buffer), self.memory_size))

        start_time = time.time()
        total_loss = 0
        ct = 0
        while ct < n_batches:
            # random sample agent episodes (sequence)
            indexs = np.random.choice(len(replay_buffer), self.batch_size, p=weight)
            train_length = 0
            to_sort = []

            # collect length and sort
            for j, index in enumerate(indexs):
                length = replay_buffer[index][-1]
                length = self._div_round(length, unroll_step)
                train_length = max(train_length, length)
                to_sort.append([index, length])
            to_sort.sort(key=lambda x: -x[1])

            # merge short episodes to long episodes (use greedy method)
            merged = [False for _ in range(batch_size)]
            rows = []
            for j in range(len(to_sort)):
                if merged[j]:
                    continue
                row = [to_sort[j][0]]
                now_len = to_sort[j][1]
                if True: # use compress
                    for k in range(j+1, batch_size):
                        if now_len + to_sort[k][1] <= train_length:
                            row.append(to_sort[k][0])
                            now_len += to_sort[k][1]
                            merged[k] = True
                    rows.append(row)
            n_rows = len(rows)

            batch_reset = np.zeros((train_length, batch_size), dtype=np.bool)
            batch_mask[:] = 0

            # copy from replay buffer to batch buffer
            for j, row in enumerate(rows):
                beg = j * max_len
                init_beg = beg
                # fill a row
                for index in row:
                    v, f, a, r, terminal, mask, x = replay_buffer[index]

                    batch_reset[(beg - init_beg)/unroll_step, j] = True
                    batch_view[beg:beg+x] = v
                    batch_feature[beg:beg+x] = f
                    batch_action[beg:beg+x] = a
                    batch_reward[beg:beg+x] = r
                    batch_terminal[beg:beg+x] = terminal
                    batch_mask[beg:beg+x] = mask

                    beg += self._div_round(x, unroll_step)

            # train steps
            for j in range((train_length + unroll_step - 1) / unroll_step):
                batch_pick[:] = False
                batch_pick[:n_rows, j * unroll_step:(j+1) * unroll_step] = True

                pick = pick_buffer[batch_pick.reshape(-1)].reshape(n_rows, unroll_step)
                next_pick = np.empty((n_rows, unroll_step + 1), dtype=np.int32)  # next pick choose one more state than pick
                next_pick[:, :unroll_step] = pick
                next_pick[:, unroll_step]  = pick[:, -1] + 1
                pick = pick.reshape(-1)
                next_pick = next_pick.reshape(-1)

                steps = len(pick) / n_rows
                assert steps > 0
                if np.sum(batch_mask[pick]) < 1:
                    continue

                batch_hidden[batch_reset[j]] = np.zeros_like(batch_hidden[0])

                batch_target = self._calc_target(batch_view[next_pick], batch_feature[next_pick],
                                                 batch_reward[pick], batch_terminal[pick], batch_hidden[:n_rows],
                                                 n_rows, steps + 1)
                ret = self.sess.run(
                    [self.train_op, self.loss, self.rnn_state],
                    feed_dict={
                        self.input_view:      batch_view[pick],
                        self.input_feature:   batch_feature[pick],
                        self.action:          batch_action[pick],
                        self.target:          batch_target,
                        self.mask:            batch_mask[pick],
                        self.state_in:        batch_hidden[:n_rows],
                        self.batch_size_ph:   n_rows,
                        self.unroll_step_ph:  steps
                })
                loss, batch_hidden[:n_rows] = ret[1], ret[2]
                total_loss += loss

                if ct % self.target_update == 0:
                    self.sess.run(self.update_target_op)

                if ct % print_every == 0:
                    print("batches %5d, mask %d/%d (%d), loss %.6f, qvalue %.6f" %
                          (ct, sum(batch_mask), n_rows * train_length, n_rows, loss, self._eval(batch_target)))
                ct += 1
                self.train_ct += 1

        total_time = time.time() - start_time
        step_average = total_time / max(1.0, (ct / 1000.0))
        print("batches: %d,  total time: %.2f,  1k average: %.2f" % (ct, total_time, step_average))

        return round(total_loss / ct if ct != 0 else 0, 6), self._eval(batch_target)
 add new samples in sample_buffer to replay buffer and train
            keep hidden state (split episode into small sequence, but keep hidden states)
            this means must train some episodes continuously not fully random.
            to use this training scheme, you should also modify self._calc_target

        Parameters
        ----------
        sample_buffer: magent.utility.EpisodesBuffer
            buffer contains samples
        print_every: int
            print log every print_every batches

        Returns
        -------
        loss: float
            bellman residual loss
        value: float
            estimated state value
        
python/magent/builtin/tf_model/drqn.py