def test_anthropic_tokenizer():
    """Test that the Anthropic tokenizer properly implements the Tokenizer protocol."""
    # Create a mock Messages object that returns a predictable token count
    mock_messages = MagicMock()
    mock_messages.count_tokens.return_value.input_tokens = 5

    # Create a mock Beta object that returns our mock messages
    mock_beta = MagicMock()
    mock_beta.messages = mock_messages

    # Create a mock client that returns our mock beta
    mock_client = MagicMock()
    mock_client.beta = mock_beta

    # Create the Anthropic instance with our mock
    anthropic_llm = Anthropic(model="claude-3-5-sonnet-20241022")
    anthropic_llm._client = mock_client

    # Test that tokenizer implements the protocol
    tokenizer = anthropic_llm.tokenizer
    assert hasattr(tokenizer, "encode")

    # Test that encode returns a list of integers
    test_text = "Hello, world!"
    tokens = tokenizer.encode(test_text)
    assert isinstance(tokens, list)
    assert all(isinstance(t, int) for t in tokens)
    assert len(tokens) == 5  # Should match our mocked token count

    # Verify the mock was called correctly
    mock_messages.count_tokens.assert_called_once_with(
        messages=[{"role": "user", "content": test_text}],
        model="claude-3-5-sonnet-20241022",
    )
Test that the Anthropic tokenizer properly implements the Tokenizer protocol.
llama-index-integrations/llms/llama-index-llms-anthropic/tests/test_llms_anthropic.py