def EncodeAsIds(self, text, process_fn=None):
        """
        encode text using text tokenizer and shift Id values for command tokens
        """
        tokenization = self.text_tokenizer.EncodeAsIds(text, process_fn=process_fn)
        tokenization.tokenization = [t + self.num_command_tokens for t in tokenization.tokenization]
        tokenization.set_command_tokens(self._command_tokens)
        return tokenization

        encode text using text tokenizer and shift Id values for command tokens
        
megatron_lm/megatron/deprecated_data_utils/tokenization.py