def get_tokens(text: str) -> list[Token]:
    """Return the tokens produced by the tokenizer."""
    return [
        Token(token.tok_name[tok_type], string, start, end)
        for tok_type, string, start, end, _ in tokenize.tokenize(text)
    ]
Return the tokens produced by the tokenizer.
tests/test_tokenize.py