def init_dict(dataset):
    """
    Initialize dictionaries of dataset, please note that '0': padding_idx, '1': start_letter.
    Finally save dictionary files locally.
    """
    tokens = get_tokenlized('dataset/{}.txt'.format(dataset))
    word_set = get_word_list(tokens)
    word2idx_dict, idx2word_dict = get_dict(word_set)

    with open('dataset/{}_wi_dict.txt'.format(dataset), 'w') as dictout:
        dictout.write(str(word2idx_dict))
    with open('dataset/{}_iw_dict.txt'.format(dataset), 'w') as dictout:
        dictout.write(str(idx2word_dict))

    print('total tokens: ', len(word2idx_dict))

    Initialize dictionaries of dataset, please note that '0': padding_idx, '1': start_letter.
    Finally save dictionary files locally.
    
utils/text_process.py