def Train(self, corpus, num_text_tokens):
        """train sentencepiece model on corpus using word frequencies"""
        self.num_text_tokens = num_text_tokens
        use_model_path = self.spm_model
        random_hash = str(random.randint(0, 2147483647))
        if use_model_path is None:
            use_model_path = random_hash
        if use_model_path.endswith('.model'):
            use_model_path = use_model_path[:use_model_path.rfind('.model')]
        input_path = use_model_path + '.tsv.' + random_hash
        line_count, maxlenline = get_corpus_freq(corpus, input_path)
        line_count = min(line_count, MAX_SENTENCEPIECE_SENTENCES)
        print('line count used as input_sentence_size ', line_count, flush=True)
        print('training sentencepiece model', flush=True)
        train_string = '--input={file_path} --model_prefix={model_prefix} --vocab_size={vocab_size}' \
            + ' --model_type={model_type} --character_coverage={character_coverage} ' \
            + '--input_sentence_size={input_sentence_size} ' \
            + '--input_format=tsv'
        train_string = train_string.format(file_path=input_path, model_prefix=use_model_path, vocab_size=num_text_tokens,
                                           model_type=self.model_type, character_coverage=self.character_coverage,
                                           input_sentence_size=int(line_count))  # , #)#,
        print("calling spm.SentencePieceTrainer.Train(%s)" % (train_string), flush=True)
        spm.SentencePieceTrainer.Train(train_string)
        os.remove(input_path)
        self.spm_model = use_model_path + '.model'
        print('sentencepiece model written to ' + self.spm_model, flush=True)
train sentencepiece model on corpus using word frequencies
megatron_lm/megatron/deprecated_data_utils/tokenization.py