def parse_data(
            self,
            data_filepath='data/',
            ontology=None,
            database=None
    ):
        """
        Parse the DSTC2 data. This function will parse the user and system logs
        and will shadow each agent (user or system) by using a dialogue state
        tracker to track and update the current state, actions taken,
        utterances produced, and ground truth.

        As user and system in this specific data collection interact via
        speech, as you can see from the ASR logs, there is no guarantee that:

        1) what the user says is what the system hears
        2) what the system understands (nlu & DST) reflects the truth

        However, the system at the time of data collection was acting upon the
        information coming through its ASR, nlu, and DST components, so to
        accurately extract the system's policy we keep track of the dialogue
        State, given the system's ASR & nlu output.

        This way we can attempt to replicate the user's and the system's
        behaviour and generate policies for both.

        Depending on the configuration, this function can use pre-trained nlu
        and nlg models to parse the utterances.

        As we do not have access to the exact ontology and database that were
        used at the time of collecting the data, some warnings will be produced
        by Plato for missing database items.

        It generates the following files:

        Plato experience logs that can be loaded into Plato to train any
        component (see runDSTC2DataParser.py for an example).

        Plato experience log from the system's side:
        path + DSTC2_system

        Plato experience log from the user's side:
        path + /DSTC2_user

        User-side dialogue_policy that is a dictionary of:
        SystemAct --> <List of User Responses with Probabilities>
        models/user_simulator/user_policy_reactive.pkl

        System-side dialogue_policy that is a dictionary of:
        UserAct --> <List of System Responses with Probabilities>
        models/camrest_policy/system_policy_reactive.pkl

        User-side dialogue_policy that is a dictionary of:
        UserDialogueState --> <List of User Responses with Probabilities>
        models/user_simulator/user_policy.pkl

        System-side dialogue_policy that is a dictionary of:
        SystemDialogueState --> <List of System Responses with Probabilities>
        models/camrest_policy/sys_policy.pkl

        File listing the user's goals found while parsing the data.
        This file can be loaded into Plato's Simulated User to sample goals
        from there instaed of generating random goals.
        Data/goals.pkl


        :param data_filepath: Path to the data:
                              <PATH_TO_DSTC2_DATA>/dstc2_traindev/data/
        :param ontology: the domain ontology
        :param database: the domain database
        :return:
        """

        # Get state encoding length
        temp_dstate = \
            SlotFillingDialogueState(
                {'slots': self.ontology.ontology['system_requestable']})
        temp_dstate.initialize()

        for (dirpath, dirnames, filenames) in os.walk(self.path):
            if not filenames or filenames[0] == '.DS_Store':
                continue

            print('Parsing files at %s' % dirpath)

            # Open files
            with open(dirpath + '/label.json') as label_file, \
                    open(dirpath + '/log.json') as log_file:
                label = json.load(label_file)
                log = json.load(log_file)

                prev_usr_act_slot = ''

                # Initialize the dialogue states
                self.DSTrackerSys.initialize()
                DStateSys = deepcopy(self.DSTrackerSys.get_state())
                DStateSys_prev = deepcopy(DStateSys)

                self.DSTrackerUsr.initialize()

                sys_dacts = []
                usr_dacts = []

                prev_usr_act_slots = ''

                # Update user dialogue state with goal information
                goal = Goal()

                constr = {}
                req = {}

                for c in label['task-information']['goal']['constraints']:
                    constr[c[0]] = DialogueActItem(c[0], Operator.EQ, c[1])

                goal.constraints = constr

                for r in label['task-information']['goal']['request-slots']:
                    req[r] = DialogueActItem(r, Operator.EQ, [])

                goal.requests = req

                # Save goal
                self.goals.append(goal)

                self.DSTrackerUsr.update_goal(goal)

                DStateUsr = deepcopy(self.DSTrackerUsr.get_state())
                DStateUsr_prev = deepcopy(DStateUsr)

                sys_turn = {}
                user_turn = {}
                prev_sys_input = ''

                # Parse each dialogue turn
                for t in range(len(label['turns'])):
                    # The system has the initiative and always starts first
                    sys_turn = log['turns'][t]
                    user_turn = label['turns'][t]

                    sys_acts = []
                    sys_slots = {'area': False,
                                 'food': False,
                                 'pricerange': False,
                                 'addr': False,
                                 'name': False,
                                 'phone': False,
                                 'postcode': False}

                    delex_utterance = \
                        self.delexicalize(
                            user_turn['transcription'],
                            user_turn['semantics']['json'],
                            sys_turn['output']['dialog-acts'])

                    usr_dacts = []

                    # Get all semantic acts
                    for udact in user_turn['semantics']['json']:
                        # TODO: THIS OVERRIDES PREVIOUS ACTS
                        user_act = udact['act']
                        user_dact = DialogueAct(user_act, [])

                        if user_act == 'bye':
                            user_terminating = True
                            udact['slots'] = [['slot']]

                        elif user_act == 'request':
                            requested = udact['slots'][0][1]

                        # For each slot-value pair
                        for slot in udact['slots']:
                            for sdact in sys_turn['output']['dialog-acts']:
                                sys_act = sdact['act']

                                dact_items = []

                                if sys_act not in sys_acts:
                                    sys_acts.append(sys_act)

                                sys_act_slot = sys_act
                                if sdact['slots']:
                                    if sdact['act'] == 'request':
                                        sys_act_slot += \
                                            '_' + sdact['slots'][0][1]
                                        ss = sdact['slots'][0][1]
                                        dact_items.append(
                                            DialogueActItem(
                                                ss,
                                                Operator.EQ,
                                                ''))
                                    else:
                                        sys_act_slot += \
                                            '_' + sdact['slots'][0][0]
                                        ss = sdact['slots'][0][0]
                                        dact_items.append(
                                            DialogueActItem(
                                                ss,
                                                Operator.EQ,
                                                sdact['slots'][0][1]))

                                    if ss:
                                        sys_slots[ss] = True

                                # Retrieve user act slot
                                if user_act == 'request':
                                    usr_act_slot = user_act + '_' + slot[1]
                                    user_dact.add_item(
                                        DialogueActItem(
                                            slot[1],
                                            Operator.EQ,
                                            ''))
                                elif user_act == 'bye':
                                    # Add underscore for consistent parsing
                                    # later
                                    usr_act_slot = user_act + '_'
                                else:
                                    usr_act_slot = user_act + '_' + slot[0]
                                    user_dact.add_item(
                                        DialogueActItem(
                                            slot[0],
                                            Operator.EQ,
                                            slot[1]))

                                # Reactive version of user policy - just
                                # reacts to system actions
                                if sys_act_slot not in \
                                        self.user_policy_reactive:
                                    self.user_policy_reactive[sys_act_slot] = \
                                        {}
                                    self.user_policy_reactive[
                                        sys_act_slot]['total_turns'] = 0
                                    self.user_policy_reactive[
                                        sys_act_slot]['dacts'] = {}
                                    self.user_policy_reactive[
                                        sys_act_slot]['responses'] = {}

                                if usr_act_slot not in \
                                        self.user_policy_reactive[
                                            sys_act_slot]['dacts']:
                                    self.user_policy_reactive[
                                        sys_act_slot][
                                        'dacts'][usr_act_slot] = 1
                                    if delex_utterance != 'UNK':
                                        self.user_policy_reactive[
                                            sys_act_slot][
                                            'responses'][delex_utterance] = 1
                                else:
                                    self.user_policy_reactive[
                                        sys_act_slot][
                                        'dacts'][usr_act_slot] += 1

                                if delex_utterance != 'UNK':
                                    if delex_utterance not in\
                                            self.user_policy_reactive[
                                                sys_act_slot]['responses']:
                                        self.user_policy_reactive[
                                            sys_act_slot][
                                            'responses'][delex_utterance] = 1
                                    else:
                                        self.user_policy_reactive[
                                            sys_act_slot][
                                            'responses'][delex_utterance] += 1

                                self.user_policy_reactive[
                                    sys_act_slot]['total_turns'] += 1

                        if user_dact not in usr_dacts:
                            usr_dacts.append(user_dact)

                    # Update system's dialogue_policy. Here we use the previous
                    # dialogue state, which is where the sys
                    # act was taken from.
                    state_enc_sys = \
                        self.encode_state(DStateSys, agent_role='system')

                    # Collapse state encoding to a number
                    dstate_idx = ''.join([str(bit) for bit in state_enc_sys])

                    sys_dacts = []

                    if 'output' in sys_turn and \
                            'dialog-acts' in sys_turn['output'] and \
                            sys_turn['output']['dialog-acts']:
                        sys_act_slots = ''
                        for sda in sys_turn['output']['dialog-acts']:
                            sys_act_slot = sda['act']
                            sys_dact = DialogueAct(sys_act_slot, [])

                            if sda['slots']:
                                if sys_act_slot == 'request':
                                    sys_act_slot += '_' + sda['slots'][0][1]
                                    sys_dact.add_item(
                                        DialogueActItem(
                                            sda['slots'][0][1],
                                            Operator.EQ,
                                            ''))
                                else:
                                    sys_act_slot += '_' + sda['slots'][0][0]
                                    sys_dact.add_item(
                                        DialogueActItem(
                                            sda['slots'][0][0],
                                            Operator.EQ,
                                            sda['slots'][0][1]))

                            sys_dacts.append(sys_dact)
                            sys_act_slots += sys_act_slot + ';'

                        # Trim last ;
                        if sys_act_slots:
                            sys_act_slots = sys_act_slots[:-1]

                        if dstate_idx not in self.system_policy:
                            self.system_policy[dstate_idx] = {}
                            self.system_policy[dstate_idx]['total_turns'] = 0
                            self.system_policy[dstate_idx]['dacts'] = {}
                            self.system_policy[dstate_idx]['responses'] = {}

                        if sys_act_slots not in\
                                self.system_policy[dstate_idx]['dacts']:
                            self.system_policy[
                                dstate_idx]['dacts'][sys_act_slots] = 1
                        else:
                            self.system_policy[
                                dstate_idx]['dacts'][sys_act_slots] += 1

                        self.system_policy[dstate_idx]['total_turns'] += 1

                        for prev_usr_act_slot in prev_usr_act_slots.split(';'):
                            if prev_usr_act_slot not in \
                                    self.system_policy_reactive:
                                self.system_policy_reactive[
                                    prev_usr_act_slot] = {}
                                self.system_policy_reactive[
                                    prev_usr_act_slot]['total_turns'] = 0
                                self.system_policy_reactive[
                                    prev_usr_act_slot]['dacts'] = {}
                                self.system_policy_reactive[
                                    prev_usr_act_slot]['responses'] = {}

                            if sys_act_slots not in \
                                    self.system_policy_reactive[
                                        prev_usr_act_slot]['dacts']:
                                self.system_policy_reactive[
                                    prev_usr_act_slot][
                                    'dacts'][sys_act_slots] = 1
                            else:
                                self.system_policy_reactive[
                                    prev_usr_act_slot][
                                    'dacts'][sys_act_slots] += 1

                            self.system_policy_reactive[
                                prev_usr_act_slot]['total_turns'] += 1

                    if self.TRAIN_LANG_2_LANG:
                        usr_dacts = \
                            self.NLUSys.process_input(
                                user_turn['transcription'], DStateSys)
                        sys_dacts = \
                            self.NLUUsr.process_input(
                                sys_turn['output']['transcript'], DStateUsr)

                    # Track the system's dialogue state. This will be relevant
                    # in the next turn.
                    # Encode DStateSys
                    DStateSys_prev = deepcopy(DStateSys)
                    self.DSTrackerSys.update_state(usr_dacts)
                    DStateSys = \
                        deepcopy(
                            self.DSTrackerSys.update_state_db(
                                self.db_lookup()))

                    # Track the user's dialogue state. This is relevant in
                    # the present turn.
                    DStateUsr_prev = deepcopy(DStateUsr)

                    # For Supervised agent this seems to help as it keeps
                    # track of the slots filled
                    # self.DSTrackerUsr.update_state(usr_dacts)

                    self.DSTrackerUsr.update_state(sys_dacts)
                    self.DSTrackerUsr.update_state_db(sys_acts=sys_dacts)
                    # self.DSTrackerUsr.update_state_sysact(usr_dacts)
                    DStateUsr = deepcopy(self.DSTrackerUsr.get_state())

                    # Encode DStateSys
                    state_enc_usr = \
                        self.encode_state(DStateUsr, agent_role='user')

                    # Collapse state encoding to a number
                    dstate_idx = ''.join([str(bit) for bit in state_enc_usr])

                    # Agent to agent version of user policy - based on state

                    # Note: It may be duplicate effort to re-iterate over
                    # the user acts. For now I let it be for clarity.

                    # Disregard empty user actions
                    if usr_dacts:
                        usr_act_slots = ''
                        for ud in usr_dacts:
                            usr_act_slot = ud.intent

                            if ud.params:
                                usr_act_slot += '_' + ud.params[0].slot

                            usr_act_slots += usr_act_slot + ';'

                        # Trim last ;
                        usr_act_slots = usr_act_slots[:-1]

                        if dstate_idx not in self.user_policy:
                            self.user_policy[dstate_idx] = {}
                            self.user_policy[dstate_idx]['total_turns'] = 0
                            self.user_policy[dstate_idx]['dacts'] = {}
                            self.user_policy[dstate_idx]['responses'] = {}

                        if usr_act_slots not in \
                                self.user_policy[dstate_idx]['dacts']:
                            self.user_policy[
                                dstate_idx]['dacts'][usr_act_slots] = 1
                        else:
                            self.user_policy[
                                dstate_idx]['dacts'][usr_act_slots] += 1

                        self.user_policy[dstate_idx]['total_turns'] += 1

                        prev_usr_act_slots = usr_act_slots

                    if not sys_acts:
                        continue

                    # Record experience

                    if bool(label['task-information']['feedback']['success']):
                        # Hack for Supervised policies that cannot handle
                        #  multiple actions
                        if prev_usr_act_slot and \
                                'request' in prev_usr_act_slot and\
                                len(sys_dacts) > 1 and\
                                sys_dacts[0].intent == 'offer':
                            ssdacts = deepcopy(sys_dacts[1:])
                        else:
                            ssdacts = deepcopy(sys_dacts)

                        t_intent, t_bio_tags = self.BIO_tag(
                                            user_turn['semantics']['json'],
                                            user_turn['transcription'],
                                            mode='sys')

                        self.recorder_sys.record(
                            DStateSys_prev,
                            DStateSys,
                            ssdacts,
                            input_utterance=prev_sys_input,
                            output_utterance=sys_turn['output']['transcript'],
                            reward=-1,
                            success=False,
                            role='system',
                            custom={'dact_str':
                                        str(sys_turn['output']['dialog-acts']),
                                    'utterance':
                                        user_turn['transcription'],
                                    'intent': t_intent,
                                    'bio_tags': t_bio_tags
                                    }
                        )

                        self.recorder_usr.record(
                            DStateUsr_prev,
                            DStateUsr,
                            deepcopy(usr_dacts),
                            input_utterance=sys_turn['output']['transcript'],
                            output_utterance=user_turn['transcription'],
                            reward=-1,
                            success=False,
                            role='user',
                            custom=str(user_turn['semantics']['json'])
                        )

                        prev_sys_input =\
                            sys_turn['input']['live']['asr-hyps'][0]['asr-hyp']

                # Record final experience (end of dialogue) - prev & current
                # states will be the same here.
                if bool(label['task-information']['feedback']['success']):
                    # Hack for Supervised policies that cannot handle
                    #  multiple actions
                    if prev_usr_act_slot and \
                            'request' in prev_usr_act_slot and \
                            len(sys_dacts) > 1 and \
                            sys_dacts[0].intent == 'offer':
                        ssdacts = deepcopy(sys_dacts[1:])
                    else:
                        ssdacts = deepcopy(sys_dacts)

                    self.recorder_sys.record(
                        DStateSys_prev,
                        DStateSys,
                        ssdacts,
                        20 if label['task-information']['feedback']['success']
                        else -20,
                        bool(label['task-information']['feedback']['success']),
                        input_utterance=prev_sys_input,
                        output_utterance='',
                        force_terminate=True)

                    self.recorder_usr.record(
                        DStateUsr_prev,
                        DStateUsr,
                        deepcopy(usr_dacts),
                        20 if label['task-information']['feedback']['success']
                        else -20,
                        bool(label['task-information']['feedback']['success']),
                        input_utterance=sys_turn['output']['transcript'],
                        output_utterance=user_turn['transcription'],
                        force_terminate=True)

        # Save data for LU, ST, and LG
        print('\n\nProcessing nlu, dst, and nlg files...')

        # sys's DST, nlg and usr's nlu
        with open(data_filepath + 'DSTC2_NLG_sys.csv', 'a') as sys_nlg_file, \
                open(data_filepath + 'DSTC2_NLU_usr.csv', 'a') \
                        as usr_nlu_file, \
                open(data_filepath + 'DST_sys.csv', 'a') as sys_dst_file:
            sys_nlg_writer = csv.writer(sys_nlg_file, delimiter=',')
            usr_nlu_writer = csv.writer(usr_nlu_file, delimiter=',')
            sys_dst_writer = csv.writer(sys_dst_file, delimiter=',')

            # Write headers
            sys_nlg_writer.writerow(
                ['dialog-acts_str', 'transcript', 'nlg_input', 'nlg_output'])

            usr_nlu_writer.writerow(['transcript', 'intent', 'iob'])

            sys_dst_writer.writerow(
                # Previous Dialogue State
                ['dst_prev_food', 'dst_prev_area',
                 'dst_prev_pricerange',

                 # Input from NLU
                 'nlu_intent',
                 'req_slot',
                 'inf_area_value',
                 'inf_food_value',
                 'inf_pricerange_value',

                 # New Dialogue State
                 'dst_food', 'dst_area', 'dst_pricerange',
                 'dst_req_slot'
                 ]
            )

            for dialogue in self.recorder_sys.dialogues:
                for sys_turn in dialogue:
                    utterance = sys_turn['output_utterance']
                    dact_str = sys_turn['custom']['dact_str'] \
                        if 'dact_str' in sys_turn['custom'] \
                        else sys_turn['custom']

                    if not utterance or not dact_str:
                        continue

                    try:
                        dialogue_acts = json.loads(dact_str.replace("'", '"'))

                    except:
                        tmp = dact_str.replace("\'", '"')
                        tmp = re.sub(r'([a-z])"([a-z])', r"\1'\2", tmp)
                        dialogue_acts = json.loads(tmp)

                    intent, iob_tags = \
                        self.BIO_tag(dialogue_acts, utterance, 'usr')
                    [delex_transcript, delex_dacts] = \
                        self.delexicalizeNLG(utterance, dialogue_acts)

                    sys_nlg_writer.writerow(
                        [dact_str, utterance, delex_dacts, delex_transcript])
                    usr_nlu_writer.writerow(
                        [utterance, ' '.join(intent), ' '.join(iob_tags)])

                    # Special cases for sys nlg
                    # If there is act_offer <name> in delex_dacts add another
                    # pair with act_offer <name> removed
                    if "act_offer <name> " in delex_dacts:
                        delex_dacts = \
                            delex_dacts.replace('act_offer <name> ', '')
                        delex_transcript = \
                            delex_transcript.replace('of <name> ', '')
                        delex_transcript = \
                            delex_transcript.replace('<name> ', 'it ')

                        sys_nlg_writer.writerow(
                            [dact_str, utterance,
                             delex_dacts, delex_transcript])

                    dst_prev_food = 'none'
                    if sys_turn['state'].slots_filled['food']:
                        dst_prev_food = sys_turn['state'].slots_filled['food']

                    dst_prev_pricerange = 'none'
                    if sys_turn['state'].slots_filled['pricerange']:
                        dst_prev_pricerange = \
                            sys_turn['state'].slots_filled['pricerange']

                    dst_prev_area = 'none'
                    if sys_turn['state'].slots_filled['area']:
                        dst_prev_area = sys_turn['state'].slots_filled['area']

                    dst_req_slot = 'none'
                    nlu_intent = set()
                    inf_area_value = 'none'
                    inf_food_value = 'none'
                    inf_pricerange_value = 'none'

                    if sys_turn['new_state'].user_acts:
                        for uact in sys_turn['new_state'].user_acts:
                            nlu_intent.add(uact.intent)

                            for p in uact.params:
                                if p.slot == 'area':
                                    inf_area_value = p.value

                                elif p.slot == 'food':
                                    inf_food_value = p.value

                                elif p.slot == 'pricerange':
                                    inf_pricerange_value = p.value

                            # Special case for requests
                            if uact.intent == 'request' and uact.params:
                                dst_req_slot = uact.params[0].slot

                        dst_food = 'none'
                        if sys_turn['new_state'].slots_filled['food']:
                            dst_food = \
                                sys_turn['new_state'].slots_filled['food']

                        dst_pricerange = 'none'
                        if sys_turn['new_state'].slots_filled['pricerange']:
                            dst_pricerange = \
                                sys_turn['new_state'].slots_filled[
                                    'pricerange']

                        dst_area = 'none'
                        if sys_turn['new_state'].slots_filled['area']:
                            dst_area = \
                                sys_turn['new_state'].slots_filled['area']

                        sys_dst_writer.writerow([dst_prev_food,
                                                 dst_prev_area,
                                                 dst_prev_pricerange,
                                                 ' '.join(nlu_intent),
                                                 dst_req_slot,
                                                 inf_area_value,
                                                 inf_food_value,
                                                 inf_pricerange_value,
                                                 dst_food,
                                                 dst_area,
                                                 dst_pricerange,
                                                 dst_req_slot])

                # Another special case for sys nlg: sys side has no bye()
                sys_nlg_writer.writerow(["[{'slots': [], 'act': 'bye'}]",
                                         'good bye', 'act_bye', 'good bye'])

        # sys's nlu and usr's DST, nlg
        with open(data_filepath + 'DSTC2_NLU_sys.csv', 'a') as sys_nlu_file, \
                open(data_filepath + 'DSTC2_NLG_usr.csv', 'a') as \
                usr_nlg_file, \
                open(data_filepath + 'DST_usr.csv', 'a') as usr_dst_file:
            sys_nlu_writer = csv.writer(sys_nlu_file, delimiter=',')
            usr_nlg_writer = csv.writer(usr_nlg_file, delimiter=',')
            usr_dst_writer = csv.writer(usr_dst_file, delimiter=',')

            # Write headers
            sys_nlu_writer.writerow(['transcript', 'intent', 'iob'])
            usr_nlg_writer.writerow(['dialog-acts_str', 'transcript',
                                     'nlg_input', 'nlg_output'])
            usr_dst_writer.writerow(['dst_prev_food', 'dst_prev_area',
                                     'dst_prev_pricerange', 'dst_intent',
                                     'dst_slot', 'dst_value', 'dst_food',
                                     'dst_area', 'dst_pricerange',
                                     'dst_req_slot'
                                     ])

            for dialogue in self.recorder_usr.dialogues:
                for usr_turn in dialogue:
                    utterance = usr_turn['output_utterance']
                    dact_str = usr_turn['custom']

                    if not utterance or not dact_str:
                        continue

                    try:
                        dialogue_acts = json.loads(dact_str.replace("'", '"'))

                    except:
                        tmp = dact_str.replace("\'", '"')
                        tmp = re.sub(r'([a-z])"([a-z])', r"\1'\2", tmp)
                        dialogue_acts = json.loads(tmp)

                    intent, iob_tags = \
                        self.BIO_tag(dialogue_acts, utterance, 'sys')
                    [delex_transcript, delex_dacts] = \
                        self.delexicalizeNLG(utterance, dialogue_acts)

                    usr_nlg_writer.writerow(
                        [dact_str, utterance, delex_dacts, delex_transcript])
                    sys_nlu_writer.writerow(
                        [utterance, ' '.join(intent), ' '.join(iob_tags)])

                    dst_prev_food = 'none'
                    if usr_turn['state'].slots_filled['food']:
                        dst_prev_food = usr_turn['state'].slots_filled['food']

                    dst_prev_pricerange = 'none'
                    if usr_turn['state'].slots_filled['pricerange']:
                        dst_prev_pricerange = usr_turn['state'].slots_filled[
                            'pricerange']

                    dst_prev_area = 'none'
                    if usr_turn['state'].slots_filled['area']:
                        dst_prev_area = usr_turn['state'].slots_filled['area']

                    dst_req_slot = 'none'
                    nlu_intent = set()
                    inf_area_value = 'none'
                    inf_food_value = 'none'
                    inf_pricerange_value = 'none'

                    if usr_turn['new_state'].user_acts:
                        for sact in usr_turn['new_state'].user_acts:
                            nlu_intent.add(sact.intent)

                            for p in sact.params:
                                if p.slot == 'area':
                                    inf_area_value = p.value

                                elif p.slot == 'food':
                                    inf_food_value = p.value

                                elif p.slot == 'pricerange':
                                    inf_pricerange_value = p.value

                            # Special case for requests
                            if sact.intent == 'request' and sact.params:
                                dst_req_slot = sact.params[0].slot

                        dst_food = 'none'
                        if usr_turn['new_state'].slots_filled['food']:
                            dst_food = \
                                usr_turn['new_state'].slots_filled['food']

                        dst_pricerange = 'none'
                        if usr_turn['new_state'].slots_filled['pricerange']:
                            dst_pricerange = \
                                usr_turn['new_state'].slots_filled[
                                    'pricerange']

                        dst_area = 'none'
                        if usr_turn['new_state'].slots_filled['area']:
                            dst_area = \
                                usr_turn['new_state'].slots_filled['area']

                        usr_dst_writer.writerow([dst_prev_food,
                                                 dst_prev_area,
                                                 dst_prev_pricerange,
                                                 ' '.join(nlu_intent),
                                                 dst_req_slot,
                                                 inf_area_value,
                                                 inf_food_value,
                                                 inf_pricerange_value,
                                                 dst_food,
                                                 dst_area,
                                                 dst_pricerange,
                                                 dst_req_slot])

        print('Done!')

        # Normalize frequencies for user policy
        for sa in self.user_policy_reactive:
            for ua in self.user_policy_reactive[sa]['dacts']:
                self.user_policy_reactive[sa]['dacts'][ua] /= \
                    self.user_policy_reactive[sa]['total_turns']

            for ur in self.user_policy_reactive[sa]['responses']:
                self.user_policy_reactive[sa]['responses'][ur] /= \
                    self.user_policy_reactive[sa]['total_turns']

        # Normalize frequencies for system policy
        for ua in self.system_policy_reactive:
            for sa in self.system_policy_reactive[ua]['dacts']:
                self.system_policy_reactive[ua]['dacts'][sa] /= \
                    self.system_policy_reactive[ua]['total_turns']

            for sr in self.system_policy_reactive[ua]['responses']:
                self.system_policy_reactive[ua]['responses'][sr] /=\
                    self.system_policy_reactive[ua]['total_turns']

        # Normalize frequencies for user calculated policy
        for state in self.user_policy:
            for ua in self.user_policy[state]['dacts']:
                self.user_policy[state]['dacts'][ua] /= \
                    self.user_policy[state]['total_turns']

        # Normalize frequencies for system calculated policy
        for state in self.system_policy:
            for sa in self.system_policy[state]['dacts']:
                self.system_policy[state]['dacts'][sa] /= \
                    self.system_policy[state]['total_turns']

        Parse the DSTC2 data. This function will parse the user and system logs
        and will shadow each agent (user or system) by using a dialogue state
        tracker to track and update the current state, actions taken,
        utterances produced, and ground truth.

        As user and system in this specific data collection interact via
        speech, as you can see from the ASR logs, there is no guarantee that:

        1) what the user says is what the system hears
        2) what the system understands (nlu & DST) reflects the truth

        However, the system at the time of data collection was acting upon the
        information coming through its ASR, nlu, and DST components, so to
        accurately extract the system's policy we keep track of the dialogue
        State, given the system's ASR & nlu output.

        This way we can attempt to replicate the user's and the system's
        behaviour and generate policies for both.

        Depending on the configuration, this function can use pre-trained nlu
        and nlg models to parse the utterances.

        As we do not have access to the exact ontology and database that were
        used at the time of collecting the data, some warnings will be produced
        by Plato for missing database items.

        It generates the following files:

        Plato experience logs that can be loaded into Plato to train any
        component (see runDSTC2DataParser.py for an example).

        Plato experience log from the system's side:
        path + DSTC2_system

        Plato experience log from the user's side:
        path + /DSTC2_user

        User-side dialogue_policy that is a dictionary of:
        SystemAct --> <List of User Responses with Probabilities>
        models/user_simulator/user_policy_reactive.pkl

        System-side dialogue_policy that is a dictionary of:
        UserAct --> <List of System Responses with Probabilities>
        models/camrest_policy/system_policy_reactive.pkl

        User-side dialogue_policy that is a dictionary of:
        UserDialogueState --> <List of User Responses with Probabilities>
        models/user_simulator/user_policy.pkl

        System-side dialogue_policy that is a dictionary of:
        SystemDialogueState --> <List of System Responses with Probabilities>
        models/camrest_policy/sys_policy.pkl

        File listing the user's goals found while parsing the data.
        This file can be loaded into Plato's Simulated User to sample goals
        from there instaed of generating random goals.
        Data/goals.pkl


        :param data_filepath: Path to the data:
                              <PATH_TO_DSTC2_DATA>/dstc2_traindev/data/
        :param ontology: the domain ontology
        :param database: the domain database
        :return:
        
plato/utilities/parser/parse_dstc2.py