def jaccard_similarity(code1: str, code2: str) -> float:
    """Compute the Jaccard similarity of two code snippets."""
    tokens1 = get_tokens(code1)
    tokens2 = get_tokens(code2)
    return len(tokens1 & tokens2) / len(tokens1 | tokens2)
Compute the Jaccard similarity of two code snippets.
dpkd/transformers/examples/research_projects/codeparrot/scripts/minhash_deduplication.py