examples/research_projects/codeparrot/scripts/minhash_deduplication.py/get_tokens
def get_tokens(code: str) -> Set[str]:
    """Tokenize a code snippet."""
    return {t for t in NON_ALPHA.split(code) if len(t.strip()) > 0}