def count_tokens_for_text(text: str, model: str = "gpt-3.5-turbo") -> int:
    """Count the number of tokens in the given text using tiktoken.

    Default model is set to "gpt-3.5-turbo". Adjust if you prefer a different
    model.
    """
    try:
        encoding = tiktoken.encoding_for_model(model)
    except KeyError:
        # Fallback to a known encoding if model not recognized
        encoding = tiktoken.get_encoding("cl100k_base")
    return len(encoding.encode(text))
Count the number of tokens in the given text using tiktoken.

    Default model is set to "gpt-3.5-turbo". Adjust if you prefer a different
    model.
    
py/migrations/versions/3efc7b3b1b3d_add_total_tokens_count.py