def _load_data(self):
        r'''Loading dataset, invoked during the initialization of :class:`SingleTurnDialog`.
        '''
        origin_data = {}
        vocab = {}

        def count_token(tokens):
            for token in tokens:
                vocab[token] = vocab[token] + 1 if token in vocab else 1

        for key in self.key_name:
            origin_data[key] = {'post': [], 'resp': [], 'kg': [], 'kg_index': []}
            datas = json.load(open("%s/%s.json" % (self._file_path, key), encoding='utf8'))

            for data in datas:
                messages = data['messages']
                kg = []
                kg_index = []
                kg_dict = {}
                for message in messages:
                    kg_index.append([])
                    if 'attrs' in message:
                        for attr in message['attrs']:
                            h = jieba.lcut(attr['zsname'])
                            r = jieba.lcut(attr['zsattrname'])
                            t = jieba.lcut(attr['zsattrvalue'])
                            k = tuple((tuple(h), tuple(r), tuple(t)))
                            if k not in kg_dict:
                                kg_dict[k] = len(kg)
                                kg.append(k)
                            kg_index[-1].append(kg_dict[k])
                            count_token(h + r + t)

                history = []
                i = 0
                nxt_sent = jieba.lcut(messages[0]['message'])
                while i + 1 < len(messages):
                    tmp_sent, nxt_sent = nxt_sent, jieba.lcut(messages[i + 1]['message'])
                    history.append(tmp_sent)
                    post = []
                    for jj in range(max(-self._num_turns + 1, -i - 1), 0):
                        post = post + history[jj] + ['<eos>', '<go>']
                    post = post[:-2]

                    count_token(nxt_sent)
                    if i == 0:
                        count_token(tmp_sent)

                    origin_data[key]['post'].append(post)
                    origin_data[key]['resp'].append(nxt_sent)
                    origin_data[key]['kg'].append(kg)
                    origin_data[key]['kg_index'].append(kg_index[i + 1])

                    i += 1

        # Important: Sort the words preventing the index changes between different runs
        vocab = sorted(list(vocab.items()), key=lambda pair: (-pair[1], pair[0]))
        left_vocab = list(filter(lambda x: x[1] >= self._min_vocab_times, vocab))
        left_vocab = list(map(lambda x: x[0], left_vocab))
        vocab_list = self.ext_vocab + left_vocab
        valid_vocab_len = len(vocab_list)
        valid_vocab_set = set(vocab_list)

        left_vocab = list(filter(lambda x: x[1] >= self._invalid_vocab_times and x[0] not in valid_vocab_set, vocab))
        vocab_list.extend(list(map(lambda x: x[0], left_vocab)))

        print("valid vocab list length = %d" % valid_vocab_len)
        print("vocab list length = %d" % len(vocab_list))

        word2id = {w: i for i, w in enumerate(vocab_list)}
        know2id = lambda line: list(map(lambda word: word2id.get(word, self.unk_id), line))
        line2id = lambda line: ([self.go_id] + list(map(lambda word: word2id[word] if word in word2id else self.unk_id, line)) + [
            self.eos_id])[:self._max_sent_length]
        knows2id = lambda lines: list(map(know2id, lines))

        data = {}
        data_size = {}
        for key in self.key_name:
            data[key] = {}
            data[key]['post'] = list(map(line2id, origin_data[key]['post']))
            data[key]['resp'] = list(map(line2id, origin_data[key]['resp']))
            data[key]['kg'] = [list(map(knows2id, kg)) for kg in origin_data[key]['kg']]
            data[key]['kg_index'] = origin_data[key]['kg_index']
            data_size[key] = len(data[key]['post'])

        return vocab_list, valid_vocab_len, data, data_size
Loading dataset, invoked during the initialization of :class:`SingleTurnDialog`.
        
benchmark/bertret/myCoTK/dataloader/single_turn_dialog.py