def sentence_tokenize(self, sent, sentence_num=0, beginning=False, ending=False):
        """tokenize sentence and get token types"""
        tokens = self.tokenizer.EncodeAsIds(sent).tokenization
        str_type = 'str' + str(sentence_num)
        token_types = [self.tokenizer.get_type(str_type).Id] * len(tokens)
        return tokens, token_types
tokenize sentence and get token types
megatron_lm/megatron/deprecated_data_utils/datasets.py