def __init__(self,
                 total_text,
                 data_root,
                 count_thresh=5,
                 max_len=-1,
                 add_special_token=True):
        """
        A class convert raw text into numbers
        :param total_text: all text for build vocab
        :param data_root: dataset root
        :param count_thresh: The number of occurrences greater than the threshold is included in the dictionary
        :param max_len: max length of a text, manually or automatically set the longest sentence in the dataset
        :param add_special_token: whether to add <start> and <end>
        """

        self.total_text = total_text
        self.data_root = data_root
        self.count_thresh = count_thresh
        self.add_special_token = add_special_token

        if not os.path.exists(os.path.join(self.data_root, 'word_dict.json')):
            vocab, length = build_vocab(self.total_text, self.data_root, self.count_thresh)
            self.word2idx = vocab
            self.idx2word = {int(v): k for k, v in vocab.items()}
        else:
            word_dict = json.load(open(os.path.join(self.data_root, 'word_dict.json'), 'r'))
            self.word2idx = word_dict['word2idx']
            self.idx2word = word_dict['idx2word']
            self.idx2word = {int(k):v for k,v in self.idx2word.items()}
            length = word_dict['length']

        self.max_len = max_len if max_len > 0 else length + 2
        self.vocab_size = len(self.word2idx)

        A class convert raw text into numbers
        :param total_text: all text for build vocab
        :param data_root: dataset root
        :param count_thresh: The number of occurrences greater than the threshold is included in the dictionary
        :param max_len: max length of a text, manually or automatically set the longest sentence in the dataset
        :param add_special_token: whether to add <start> and <end>
        
torchmm/datasets/preprocess/base_tokenizer.py