def _load_data(self):
        r'''Loading dataset, invoked during the initialization of :class:`SingleTurnDialog`.
        '''
        origin_data = {}
        vocab = {}

        def count_token(tokens):
            for token in tokens:
                vocab[token] = vocab[token] + 1 if token in vocab else 1

        for key in self.key_name:
            origin_data[key] = {'post': [], 'resp': []}
            datas = json.load(open("%s/%s.json" % (self._file_path, key)))

            for data in datas:
                messages = data['messages']
                i = 0
                if key != 'test':
                    resp_sent = []
                    while i + 1 < len(messages):

                        if i == 0:
                            tmp_sent = jieba.lcut(messages[i]['message'])
                            resp_sent = tmp_sent

                            count_token(tmp_sent)
                            if 'attrs' in messages[0]:
                                for attr in messages[0]['attrs']:
                                    h = jieba.lcut(attr['zsname'])
                                    r = jieba.lcut(attr['zsattrname'])
                                    t = jieba.lcut(attr['zsattrvalue'])
                                    count_token(h + r + t)

                        nxt_sent = jieba.lcut(messages[i + 1]['message'])
                        resp_sent = resp_sent + ['<eos>'] + nxt_sent

                        count_token(nxt_sent)
                        if 'attrs' in messages[i + 1]:
                            for attr in messages[i + 1]['attrs']:
                                h = jieba.lcut(attr['zsname'])
                                r = jieba.lcut(attr['zsattrname'])
                                t = jieba.lcut(attr['zsattrvalue'])
                                count_token(h + r + t)

                        i += 1


                    origin_data[key]['post'].append([])
                    origin_data[key]['resp'].append(resp_sent)

                else:
                    post_sent = []
                    while i + 1 < len(messages):

                        if i == 0:
                            post_sent = jieba.lcut(messages[0]['message'])
                            count_token(post_sent)
                            if 'attrs' in messages[0]:
                                for attr in messages[0]['attrs']:
                                    h = jieba.lcut(attr['zsname'])
                                    r = jieba.lcut(attr['zsattrname'])
                                    t = jieba.lcut(attr['zsattrvalue'])
                                    count_token(h + r + t)

                        nxt_sent = jieba.lcut(messages[i + 1]['message'])
                        origin_data[key]['post'].append(post_sent)
                        origin_data[key]['resp'].append(nxt_sent)

                        post_sent = post_sent + ['<eos>'] + nxt_sent

                        count_token(nxt_sent)
                        if 'attrs' in messages[i + 1]:
                            for attr in messages[i + 1]['attrs']:
                                h = jieba.lcut(attr['zsname'])
                                r = jieba.lcut(attr['zsattrname'])
                                t = jieba.lcut(attr['zsattrvalue'])
                                count_token(h + r + t)

                        i += 1

        # Important: Sort the words preventing the index changes between different runs
        vocab = sorted(list(vocab.items()), key=lambda pair: (-pair[1], pair[0]))
        left_vocab = list(filter(lambda x: x[1] >= self._min_vocab_times, vocab))
        left_vocab = list(map(lambda x: x[0], left_vocab))
        vocab_list = self.ext_vocab + left_vocab
        valid_vocab_len = len(vocab_list)
        valid_vocab_set = set(vocab_list)

        left_vocab = list(filter(lambda x: x[1] >= self._invalid_vocab_times and x[0] not in valid_vocab_set, vocab))
        vocab_list.extend(list(map(lambda x: x[0], left_vocab)))

        print("valid vocab list length = %d" % valid_vocab_len)
        print("vocab list length = %d" % len(vocab_list))

        word2id = {w: i for i, w in enumerate(vocab_list)}
        line2id = lambda line: ([self.go_id] +
                                  list(map(lambda word: word2id[word] if word in word2id else self.unk_id, line)) +
                                  [self.eos_id])[:self._max_sent_length]
        line2id_post = lambda line: ([self.go_id] +
                                  list(map(lambda word: word2id[word] if word in word2id else self.unk_id, line))
                                  )[:self._max_sent_length]
        line2id_resp = lambda line: ([self.eos_id] +
                                  list(map(lambda word: word2id[word] if word in word2id else self.unk_id, line)) +
                                  [self.eos_id])[:self._max_sent_length]

        data = {}
        data_size = {}
        for key in self.key_name:
            data[key] = {}
            if key != 'test':
                data[key]['post'] = list(map(line2id, origin_data[key]['post']))
                data[key]['resp'] = list(map(line2id, origin_data[key]['resp']))
            else:
                data[key]['post'] = list(map(line2id_post, origin_data[key]['post']))
                data[key]['resp'] = list(map(line2id_resp, origin_data[key]['resp']))
            data_size[key] = len(data[key]['post'])

            vocab = list(chain(*(origin_data[key]['post'] + origin_data[key]['resp'])))
            vocab_num = len(vocab)
            oov_num = len(list(filter(lambda word: word not in word2id, vocab)))
            invalid_num = len(list(filter(lambda word: word not in valid_vocab_set, vocab))) - oov_num
            length = list(map(len, origin_data[key]['post'] + origin_data[key]['resp']))
            cut_num = np.sum(np.maximum(np.array(length) - self._max_sent_length + 1, 0))
            print("%s set. invalid rate: %f, unknown rate: %f, max length before cut: %d, cut word rate: %f" % \
                  (key, invalid_num / vocab_num, oov_num / vocab_num, max(length), cut_num / vocab_num))

        return vocab_list, valid_vocab_len, data, data_size
Loading dataset, invoked during the initialization of :class:`SingleTurnDialog`.
        
benchmark/memseq2seq/myCoTK/dataloader/single_turn_dialog.py