def vocab_extend(corpus, raw_vocab, output_filename, interval=10000 , threshold = 0.01):
    """
    @description  : The function to get the incremental vocabulary for 
    
    @param  :
    
    @Returns  :
    
    """
    
    
    documents = []
    for line in open(corpus, "r",encoding='utf-8'):
        line = line.replace('\n','')
        if len(line) < 5:
            continue
        documents.append(line)
    print("docunments: "+str(len(documents)))
    token_counts = tokenizer.corpus_token_counts(
        corpus, corpus_max_lines = 4400000,
        split_on_newlines = True, additional_chars="", do_lower_case=True)
    lines = open(raw_vocab, 'r', encoding='utf-8').readlines()
    lines = [s.strip() for s in lines if len(s) > 0]
    reserved_tokens = lines
    random.shuffle(documents)
    origin_size = (len(reserved_tokens) // interval) * interval
    pre_lm = compute_language_model(documents, raw_vocab)
    print("origin_size: " + str(origin_size))
    print("pre_lm: "+ str(pre_lm))
    target_size = origin_size
    while True:
        target_size = target_size + interval
        _, temp_vocab = build_target_size_vocab(token_counts, reserved_tokens, target_size)
        now_lm = compute_language_model(documents, temp_vocab)
        print('now_lm: '+ str(now_lm))
        delta = (pre_lm - now_lm)/pre_lm
        print('delta: ' + str(delta))
        if delta <= threshold:
           merge_output_file_with_bert_vocab(output_filename, raw_vocab, temp_vocab)
           break
        pre_lm = now_lm

    @description  : The function to get the incremental vocabulary for 
    
    @param  :
    
    @Returns  :
    
    
adalm/incr_bpe/vocab_extend.py