def assets_tokenizer_helper(name: str, **kwargs: Any):
    """Load a tokenizer from the assets directory."""
    transformers = pytest.importorskip('transformers')

    download_tokenizers_files()

    assets_dir = assets_path()
    tokenizer_path = os.path.join(assets_dir, name)

    # Load the tokenizer
    hf_tokenizer = transformers.AutoTokenizer.from_pretrained(
        tokenizer_path,
        **kwargs,
    )
    return hf_tokenizer
Load a tokenizer from the assets directory.
tests/fixtures/models.py