def load_test_dict(dataset):
    """Build test data dictionary, extend from train data. For the classifier."""
    word2idx_dict, idx2word_dict = load_dict(dataset)  # train dict
    # tokens = get_tokenlized('dataset/testdata/{}_clas_test.txt'.format(dataset))
    tokens = get_tokenlized('dataset/testdata/{}_test.txt'.format(dataset))
    word_set = get_word_list(tokens)
    index = len(word2idx_dict)  # current index

    # extend dict with test data
    for word in word_set:
        if word not in word2idx_dict:
            word2idx_dict[word] = str(index)
            idx2word_dict[str(index)] = word
            index += 1
    return word2idx_dict, idx2word_dict
Build test data dictionary, extend from train data. For the classifier.
utils/text_process.py