def SetTokenizer(self, tokenizer):
        """
        logic to set and remove (set to None) tokenizer.
        combines preprocessing/tokenization into one callable.
        """
        if tokenizer is None:
            if not hasattr(self, '_tokenizer'):
                self._tokenizer = tokenizer
        else:
            self._tokenizer = tokenizer
        self.map_fn = ProcessorTokenizer(tokenizer, self.process_fn)

        logic to set and remove (set to None) tokenizer.
        combines preprocessing/tokenization into one callable.
        
megatron_lm/megatron/deprecated_data_utils/lazy_loader.py