def __init__(self, config):
        """this function is borrowed from
        `transformers/data/data_collator.DataCollatorForLanguageModeling`"""
        self.mlm_probability = 0.15
        if config.mlm_probability is not None:
            self.mlm_probability = config.mlm_probability
        self.bert_name = config.bert_name
        # [CLS] is used as bos_token and [SEP] is used as eos_token.
        # https://huggingface.co/transformers/master/model_doc/bertgeneration.html
        from transformers import AutoTokenizer
        self.tokenizer = AutoTokenizer.from_pretrained(
            self.bert_name, bos_token="[CLS]", eos_token="[SEP]")
        self.textgen = TextGenerationProcessor(self.tokenizer)
this function is borrowed from
        `transformers/data/data_collator.DataCollatorForLanguageModeling`
structured_prompting/fairseq-version/fairseq/examples/MMPT/mmpt/processors/how2processor.py