def step(self, exploration):
        """
        Run the environment for one step.
        If the episode ends, store the entire episode to the replay memory.
        """
        old_s = self._current_ob
        if self.rng.rand() <= exploration:
            act = self.rng.choice(range(self.num_actions))
        else:
            history = self.recent_state()
            history.append(old_s)
            history = np.stack(history, axis=-1)  # state_shape + (Hist,)

            # assume batched network
            history = np.expand_dims(history, axis=0)
            q_values = self.predictor(history)[0][0]  # this is the bottleneck
            act = np.argmax(q_values)

        self._current_ob, reward, isOver, info = self.player.step(act)
        self._current_game_score.feed(reward)
        self._current_episode.append(Experience(old_s, act, reward, isOver))

        if isOver:
            flush_experience = True
            if 'ale.lives' in info:  # if running Atari, do something special
                if info['ale.lives'] != 0:
                    # only record score and flush experience
                    # when a whole game is over (not when an episode is over)
                    flush_experience = False
            self.player.reset()

            if flush_experience:
                self.total_scores.append(self._current_game_score.sum)
                self._current_game_score.reset()

                # Ensure that the whole episode of experience is continuous in the replay buffer
                with self.memory.writer_lock:
                    for exp in self._current_episode:
                        self.memory.append(exp)
                self._current_episode.clear()

        Run the environment for one step.
        If the episode ends, store the entire episode to the replay memory.
        
examples/DeepQNetwork/expreplay.py