def calculate(self, state, actions, goal=None, force_terminal=False,
                  agent_role='system'):
        """
        Calculate the reward based on whether the action taken advanced the
        goal or not. For example, if the action resulted in filling one more
        slot the conversation advanced towards the goal.

        :param state: the current state
        :param actions: the action taken from the current state
        :param goal: the agent's goal, used to assess success
        :param force_terminal: force state to be terminal
        :param agent_role: the role of the agent
        :return: a number, representing the calculated reward
        """

        if goal is None:
            print('Warning: SlotFillingGoalAdvancementReward() called without '
                  'a goal.')
            return -1, False, False

        elif self.prev_state is None or self.prev_goal is None:
            reward = 1

        # Check if the goal has been advanced
        else:
            # If the new state has more slots filled than the old one
            if sum([1 if self.prev_state.slots_filled[s] else 0 for s in
                    self.prev_state.slots_filled]) < \
                    sum([1 if state.slots_filled[s] else 0 for s in
                         state.slots_filled]):
                reward = 1

            # Or if the new state has more requests filled than the old one
            elif sum([1 if self.prev_goal.actual_requests[r] else 0 for r in
                      self.prev_goal.actual_requests]) < \
                    sum([1 if goal.actual_requests[r] else 0 for r in
                         goal.actual_requests]):
                reward = 1

            # Or if the system made a request for an unfilled slot?
        
            else:
                reward = -1

        success = False
        if state.is_terminal() or force_terminal:
            # Check that an offer has actually been made
            if state.system_made_offer:
                success = True

                # Check that the item offered meets the user's constraints
                for constr in goal.constraints:
                    if goal.ground_truth:
                        # Multi-agent case
                        if goal.ground_truth[constr] != \
                                goal.constraints[constr].value:
                            success = False
                            break

                    elif state.item_in_focus:
                        # Single-agent case
                        if state.item_in_focus[constr] != \
                                goal.constraints[constr].value:
                            success = False
                            break

        self.prev_state = deepcopy(state)
        self.prev_goal = deepcopy(goal)

        task_success = None
        if agent_role == 'system':
            task_success = True
            # We don't care for slots that are not in the goal constraints
            for slot in goal.constraints:
                # If the system proactively informs about a slot the user has
                # not yet put a constraint upon,
                # the user's DState is updated accordingly and the user would
                # not need to put that constraint.
                if goal.ground_truth:
                    if goal.ground_truth[slot] != \
                            goal.constraints[slot].value and \
                            goal.constraints[slot].value != 'dontcare':
                        task_success = False

                # Fall back to the noisier signal, that is the slots filled.
                elif state.slots_filled[slot] != \
                        goal.constraints[slot].value and \
                        goal.constraints[slot].value != 'dontcare':
                    task_success = False

            for req in goal.requests:
                if not goal.requests[req].value:
                    task_success = False

        return reward, success, task_success

        Calculate the reward based on whether the action taken advanced the
        goal or not. For example, if the action resulted in filling one more
        slot the conversation advanced towards the goal.

        :param state: the current state
        :param actions: the action taken from the current state
        :param goal: the agent's goal, used to assess success
        :param force_terminal: force state to be terminal
        :param agent_role: the role of the agent
        :return: a number, representing the calculated reward
        
plato/agent/component/dialogue_policy/reinforcement_learning/reward_function.py