def get_prefix_from_len(sentence, bpe_symbol, prefix_len):
    """get the prefix of sentence with bpe, with prefix len in terms of words, not bpe tokens"""
    bpe_count = sum([bpe_symbol.strip(" ") in t for t in sentence[:prefix_len]])
    if bpe_count == 0:
        return sentence[:prefix_len]
    else:
        return sentence[:prefix_len] + get_prefix_from_len(
            sentence[prefix_len:], bpe_symbol, bpe_count
        )
get the prefix of sentence with bpe, with prefix len in terms of words, not bpe tokens
understand_icl/fairseq/examples/noisychannel/rerank_utils.py