def truncate_and_pad(
    tokenizer: BaseTokenizer,
    max_length: int,
    stride: int,
    strategy: str,
    pad_to_max_length: bool,
    padding_side: str,
    pad_token_id: int,
    pad_token_type_id: int,
    pad_token: str,
):
    """
    This contextmanager is in charge of defining the truncation and the padding strategies and then
    restore the tokenizer settings afterwards.

    This contextmanager assumes the provider tokenizer has no padding / truncation strategy
    before the managed section. If your tokenizer set a padding / truncation strategy before,
    then it will be reset to no padding/truncation when exiting the managed section.

    :param tokenizer:
    :param max_length:
    :param stride:
    :param strategy:
    :param pad_to_max_length:
    :param padding_side:
    :param pad_token_id:
    :param pad_token_type_id:
    :param pad_token:
    :return:
    """

    # Handle all the truncation and padding stuff
    if max_length is not None:
        tokenizer.enable_truncation(max_length, stride=stride, strategy=strategy)

    if pad_to_max_length and (pad_token and pad_token_id >= 0):
        tokenizer.enable_padding(
            max_length=max_length,
            direction=padding_side,
            pad_id=pad_token_id,
            pad_type_id=pad_token_type_id,
            pad_token=pad_token,
        )
    elif pad_to_max_length:
        logger.warning(
            "Disabled padding because no padding token set (pad_token: {}, pad_token_id: {}).\n"
            "To remove this error, you can add a new pad token and then resize model embedding:\n"
            "\ttokenizer.pad_token = '<PAD>'\n\tmodel.resize_token_embeddings(len(tokenizer))".format(
                pad_token, pad_token_id
            )
        )

    yield

    if max_length is not None:
        tokenizer.no_truncation()

    if pad_to_max_length and (pad_token and pad_token_id >= 0):
        tokenizer.no_padding()

    This contextmanager is in charge of defining the truncation and the padding strategies and then
    restore the tokenizer settings afterwards.

    This contextmanager assumes the provider tokenizer has no padding / truncation strategy
    before the managed section. If your tokenizer set a padding / truncation strategy before,
    then it will be reset to no padding/truncation when exiting the managed section.

    :param tokenizer:
    :param max_length:
    :param stride:
    :param strategy:
    :param pad_to_max_length:
    :param padding_side:
    :param pad_token_id:
    :param pad_token_type_id:
    :param pad_token:
    :return:
    
xtune/src/transformers/tokenization_utils.py