utils/create_dummy_models.py/convert_processors
def convert_processors(processors, tiny_config, output_folder, result):
    """Change a processor to work with smaller inputs.

    For tokenizers, we try to reduce their vocabulary size.

    For feature extractor, we use smaller image size or change
    other attributes using the values from `tiny_config`. See `convert_feature_extractor`.

    This method should not fail: we catch the errors and put them in `result["warnings"]` with descriptive messages.
    """

    tokenizers = []
    feature_extractors = []
    for processor in processors:
        if isinstance(processor, PreTrainedTokenizerBase):
            tokenizers.append(processor)
        elif isinstance(processor, BaseImageProcessor):
            feature_extractors.append(processor)
        elif isinstance(processor, FeatureExtractionMixin):
            feature_extractors.append(processor)
        elif isinstance(processor, ProcessorMixin):
            # Currently, we only have these 2 possibilities
            tokenizers.append(processor.tokenizer)
            if hasattr(processor, "image_processor"):
                feature_extractors.append(processor.image_processor)
            elif hasattr(processor, "feature_extractor"):
                feature_extractors.append(processor.feature_extractor)

    # check the built processors have the unique type
    num_types = len({x.__class__.__name__ for x in feature_extractors})
    if num_types >= 2:
        raise ValueError(f"`feature_extractors` should contain at most 1 type, but it contains {num_types} types!")
    num_types = len({x.__class__.__name__.replace("Fast", "") for x in tokenizers})
    if num_types >= 2:
        raise ValueError(f"`tokenizers` should contain at most 1 tokenizer type, but it contains {num_types} types!")

    fast_tokenizer = None
    slow_tokenizer = None
    for tokenizer in tokenizers:
        if isinstance(tokenizer, PreTrainedTokenizerFast):
            if fast_tokenizer is None:
                fast_tokenizer = tokenizer
                try:
                    # Wav2Vec2ForCTC , ByT5Tokenizer etc. all are already small enough and have no fast version that can
                    # be retrained
                    if fast_tokenizer.vocab_size > TARGET_VOCAB_SIZE:
                        fast_tokenizer = convert_tokenizer(tokenizer)
                except Exception:
                    result["warnings"].append(
                        (
                            f"Failed to convert the fast tokenizer for {fast_tokenizer.__class__.__name__}.",
                            traceback.format_exc(),
                        )
                    )
                    continue
        elif slow_tokenizer is None:
            slow_tokenizer = tokenizer

    # Make sure the fast tokenizer can be saved
    if fast_tokenizer:
        try:
            fast_tokenizer.save_pretrained(output_folder)
        except Exception:
            result["warnings"].append(
                (
                    f"Failed to save the fast tokenizer for {fast_tokenizer.__class__.__name__}.",
                    traceback.format_exc(),
                )
            )
            fast_tokenizer = None

    # Make sure the slow tokenizer (if any) corresponds to the fast version (as it might be converted above)
    if fast_tokenizer:
        try:
            slow_tokenizer = AutoTokenizer.from_pretrained(output_folder, use_fast=False)
        except Exception:
            result["warnings"].append(
                (
                    f"Failed to load the slow tokenizer saved from {fast_tokenizer.__class__.__name__}.",
                    traceback.format_exc(),
                )
            )
            # Let's just keep the fast version
            slow_tokenizer = None

    # If the fast version can't be created and saved, let's use the slow version
    if not fast_tokenizer and slow_tokenizer:
        try:
            slow_tokenizer.save_pretrained(output_folder)
        except Exception:
            result["warnings"].append(
                (
                    f"Failed to save the slow tokenizer for {slow_tokenizer.__class__.__name__}.",
                    traceback.format_exc(),
                )
            )
            slow_tokenizer = None

    # update feature extractors using the tiny config
    try:
        feature_extractors = [convert_feature_extractor(p, tiny_config) for p in feature_extractors]
    except Exception:
        result["warnings"].append(
            (
                "Failed to convert feature extractors.",
                traceback.format_exc(),
            )
        )
        feature_extractors = []

    if hasattr(tiny_config, "max_position_embeddings") and tiny_config.max_position_embeddings > 0:
        if fast_tokenizer is not None:
            if fast_tokenizer.__class__.__name__ in ["RobertaTokenizerFast", "XLMRobertaTokenizerFast"]:
                fast_tokenizer.model_max_length = tiny_config.max_position_embeddings - 2
            else:
                fast_tokenizer.model_max_length = tiny_config.max_position_embeddings
        if slow_tokenizer is not None:
            if slow_tokenizer.__class__.__name__ in ["RobertaTokenizer", "XLMRobertaTokenizer"]:
                slow_tokenizer.model_max_length = tiny_config.max_position_embeddings - 2
            else:
                slow_tokenizer.model_max_length = tiny_config.max_position_embeddings

    processors = [fast_tokenizer, slow_tokenizer] + feature_extractors
    processors = [p for p in processors if p is not None]
    for p in processors:
        p.save_pretrained(output_folder)

    return processors