def assert_tokenizes(text: str, tokens: list[Token]) -> None:
    """Assert that the tokenizer produces the expected tokens."""
    actual_tokens = get_tokens(text)
    assert actual_tokens == tokens
Assert that the tokenizer produces the expected tokens.
tests/test_tokenize.py