def _load_data(self):
        r'''Loading dataset, invoked during the initialization of :class:`SingleTurnDialog`.
        '''
        origin_data = {}
        vocab = {}

        def count_token(tokens):
            for token in tokens:
                vocab[token] = vocab[token] + 1 if token in vocab else 1

        for key in self.key_name:
            origin_data[key] = {'post': [], 'resp': []}
            datas = json.load(open("%s/%s.json" % (self._file_path, key)))

            for data in datas:
                history = []
                messages = data['messages']
                i = 0
                nxt_sent = jieba.lcut(messages[0]['message'])
                while i + 1 < len(messages):
                    tmp_sent, nxt_sent = nxt_sent, jieba.lcut(messages[i + 1]['message'])
                    history.append(tmp_sent)
                    post = []
                    for jj in range(max(-self._num_turns+1, -i-1), 0):
                        post = post + history[jj] + ['<eos>', '<go>']
                    post = post[:-2]
                    origin_data[key]['post'].append(post)
                    origin_data[key]['resp'].append(nxt_sent)

                    count_token(nxt_sent)
                    if 'attrs' in messages[i + 1]:
                        for attr in messages[i + 1]['attrs']:
                            h = jieba.lcut(attr['zsname'])
                            r = jieba.lcut(attr['zsattrname'])
                            t = jieba.lcut(attr['zsattrvalue'])
                            count_token(h + r + t)

                    if i == 0:
                        count_token(tmp_sent)
                        if 'attrs' in messages[0]:
                            for attr in messages[0]['attrs']:
                                h = jieba.lcut(attr['zsname'])
                                r = jieba.lcut(attr['zsattrname'])
                                t = jieba.lcut(attr['zsattrvalue'])
                                count_token(h + r + t)

                    i += 1

        # Important: Sort the words preventing the index changes between different runs
        vocab = sorted(list(vocab.items()), key=lambda pair: (-pair[1], pair[0]))
        left_vocab = list(filter(lambda x: x[1] >= self._min_vocab_times, vocab))
        left_vocab = list(map(lambda x: x[0], left_vocab))
        vocab_list = self.ext_vocab + left_vocab
        valid_vocab_len = len(vocab_list)
        valid_vocab_set = set(vocab_list)

        left_vocab = list(filter(lambda x: x[1] >= self._invalid_vocab_times and x[0] not in valid_vocab_set, vocab))
        vocab_list.extend(list(map(lambda x: x[0], left_vocab)))

        print("valid vocab list length = %d" % valid_vocab_len)
        print("vocab list length = %d" % len(vocab_list))

        word2id = {w: i for i, w in enumerate(vocab_list)}
        line2id = lambda line: ([self.go_id] +
                    list(map(lambda word: word2id[word] if word in word2id else self.unk_id, line)) +
                    [self.eos_id])[:self._max_sent_length]

        data = {}
        data_size = {}
        for key in self.key_name:
            data[key] = {}

            data[key]['post'] = list(map(line2id, origin_data[key]['post']))
            data[key]['resp'] = list(map(line2id, origin_data[key]['resp']))
            data_size[key] = len(data[key]['post'])

            vocab = list(chain(*(origin_data[key]['post'] + origin_data[key]['resp'])))
            vocab_num = len(vocab)
            oov_num = len(list(filter(lambda word: word not in word2id, vocab)))
            invalid_num = len(list(filter(lambda word: word not in valid_vocab_set, vocab))) - oov_num
            length = list(map(len, origin_data[key]['post'] + origin_data[key]['resp']))
            cut_num = np.sum(np.maximum(np.array(length) - self._max_sent_length + 1, 0))
            print("%s set. invalid rate: %f, unknown rate: %f, max length before cut: %d, cut word rate: %f" %
                  (key, invalid_num / vocab_num, oov_num / vocab_num, max(length), cut_num / vocab_num))

        return vocab_list, valid_vocab_len, data, data_size
Loading dataset, invoked during the initialization of :class:`SingleTurnDialog`.
        
benchmark/seq2seq/myCoTK/dataloader/single_turn_dialog.py