def get_num_bpe_tokens_from_len(sentence, bpe_symbol, prefix_len):
    """given a prefix length in terms of words, return the number of bpe tokens"""
    prefix = get_prefix_no_bpe(sentence, bpe_symbol, prefix_len)
    assert len(remove_bpe(prefix, bpe_symbol).split()) <= prefix_len
    return len(prefix.split(" "))
given a prefix length in terms of words, return the number of bpe tokens
structured_prompting/fairseq-version/fairseq/examples/noisychannel/rerank_utils.py