def html_str_to_token_list(
    seq: str, splitter: Optional[tk.pre_tokenizers.PreTokenizer] = None
) -> List[str]:
    """Convert decode output (str) to a list of tokens for constructing html table code"""

    # Work for no <eos>
    seq = seq.split("<eos>")[0]
    token_black_list = ["<eos>", "<pad>"]  # Assuming TASK_TOKENS is defined elsewhere
    for token in token_black_list:
        seq = seq.replace(token, "")

    if splitter is None:
        splitter = tk.pre_tokenizers.Split(pattern=" ", behavior="contiguous")

    pre_tokenized_seq = splitter.pre_tokenize_str(seq)

    filtered_tokens = filter_tokens(pre_tokenized_seq)

    return filtered_tokens
Convert decode output (str) to a list of tokens for constructing html table code
src/openparse/tables/unitable/utils.py