def fast_bert(model, dtype=torch.float, optimizer=None, unpad=False):
    r"""
    Use TPP to speedup training/inference. fast_bert API is still a prototype
    feature and now only optimized for bert model.

    Args:
        model (torch.nn.Module): User model to apply optimizations on.
        dtype (torch.dtype): Only works for ``torch.bfloat16`` and ``torch.float`` .
            The default value is torch.float.
        optimizer (torch.optim.Optimizer): User optimizer to apply optimizations
            on, such as SGD. The default value is ``None``, meaning inference case.
        unpad(bool): Unpad the squence to reduce the sparsity.
        seed(string): The seed used for the libxsmm kernel. In general it should be same
            to the torch.seed

    .. note::

        Currently ``ipex.fast_bert`` API is well optimized for training tasks.
        It works for inference tasks, though, please use the ``ipex.optimize``
        API with TorchScript to achieve the peak performance.

    .. warning::

        Please invoke ``fast_bert`` function AFTER loading weights to model via
        ``model.load_state_dict(torch.load(PATH))``.

    .. warning::

        This API can't be used when you have applied the ``ipex.optimize``.

    .. warning::

        Please invoke ``optimize`` function BEFORE invoking DDP in distributed
        training scenario.

    Examples:

        >>> # bfloat16 inference case.
        >>> model = ...
        >>> model.load_state_dict(torch.load(PATH))
        >>> model.eval()
        >>> optimized_model = ipex.fast_bert(model, dtype=torch.bfloat16)
        >>> # running evaluation step.
        >>> # bfloat16 training case.
        >>> optimizer = ...
        >>> model.train()
        >>> optimized_model, optimized_optimizer = ipex.fast_bert(model, dtype=torch.bfloat16,
                optimizer=optimizer, unpad=True, seed=args.seed)
        >>> # running training step.

    """
    # tpp bert optimization depends on the transformers repo to implementate the related module
    installed_pkg = {dist.metadata["Name"].lower() for dist in distributions()}
    min_version = "4.6.0"
    max_version = "4.51.3"
    if "transformers" not in installed_pkg:
        raise RuntimeError(
            "Please installed the transformers with version: between {} and {}".format(
                min_version, max_version
            )
        )

    import transformers
    from packaging import version

    trans_version = transformers.__version__
    if version.parse(trans_version) < version.parse(min_version) or version.parse(
        trans_version
    ) > version.parse(max_version):
        raise RuntimeError(
            "Please installed the transformers with version: between {} and {} while now transformers== {}".format(
                min_version, max_version, trans_version
            )
        )
    position_ids_persistent = False
    if version.parse(trans_version) < version.parse("4.31.0"):
        position_ids_persistent = True
    PT_OPTIMIZER_TO_TPP_OPTIMIZER = {
        torch.optim.AdamW: AdamW,
        torch.optim.SGD: SGD,
    }
    if hasattr(transformers.optimization, "AdamW"):
        PT_OPTIMIZER_TO_TPP_OPTIMIZER[transformers.optimization.AdamW] = AdamW
    if dtype not in (
        torch.float,
        torch.bfloat16,
    ):
        raise ValueError("TPP only supports torch.float and torch.bfloat16.")

    # setup the seed for libxsmm (can be only positive int value) which will imapct some ops using seed. e.g., dropout
    try:
        torch_ipex_cpp.xsmm_manual_seed(
            torch.tensor(torch.initial_seed()).to(torch.int32).abs().item()
        )
    except BaseException:
        logger.warning(
            "Set seed failed for libxsmm which may impact the training loss, you can call "
            + "torch.manual_seed(N) before invoking fast_bert."
        )
    # replace the original transfomers module object with tpp module which has the same functionality but with more
    # operator fusion optimization
    new_model = copy.deepcopy(model)
    global layer_use_bf16
    layer_use_bf16 = True if dtype == torch.bfloat16 else False
    if unpad:
        unpad = True
    else:
        unpad = False
    if isinstance(model, transformers.models.bert.modeling_bert.BertModel):
        assert isinstance(
            new_model.embeddings, transformers.models.bert.modeling_bert.BertEmbeddings
        )
        new_model.embeddings = BertEmbeddings(
            model.config, position_ids_persistent=position_ids_persistent
        )
        assert isinstance(
            new_model.encoder, transformers.models.bert.modeling_bert.BertEncoder
        )
        new_model.encoder = BertEncoder(model.config)
    elif hasattr(model, "bert") and isinstance(
        model.bert, transformers.models.bert.modeling_bert.BertModel
    ):
        assert isinstance(
            new_model.bert.embeddings,
            transformers.models.bert.modeling_bert.BertEmbeddings,
        )
        new_model.bert.embeddings = BertEmbeddings(
            model.bert.config, position_ids_persistent=position_ids_persistent
        )
        assert isinstance(
            new_model.bert.encoder, transformers.models.bert.modeling_bert.BertEncoder
        )
        new_model.bert.encoder = BertEncoder(model.bert.config)
    else:
        logger.warning(
            "fast_bert only supports instance of transformers.models.bert.modeling_bert.BertModel",
            _type=WarningType.NotSupported,
        )
        return model, optimizer
    new_model.load_state_dict(
        model.state_dict()
    )  # copy the original params into the tpp module
    block(new_model)  # get block format weights/bias
    if optimizer is None:
        logger.warning(
            "Currently ipex.fast_bert API is well optimized for training tasks. It works for inference tasks, "
            + "though, please use the ipex.optimize API with TorchScript to achieve the peak performance.",
            _type=WarningType.NotSupported,
        )
        return new_model
    # replace the original pytorch/transformer optimizer with tpp optimizer for SGD/AdamW
    # keep the original optimizer state and replace the params with the blocked tpp params
    param_pair = {}
    for param_ori, param_tpp in zip(model.parameters(), new_model.parameters()):
        param_pair[param_ori] = param_tpp
    if type(optimizer) not in PT_OPTIMIZER_TO_TPP_OPTIMIZER:
        logger.warning(
            "Still return the origin optimize, the fast_bert can only replace the SGD, AdamW optimizer",
            _type=WarningType.NotSupported,
        )
        new_optimizer = optimizer
    else:
        new_optimizer = PT_OPTIMIZER_TO_TPP_OPTIMIZER[type(optimizer)]([{"params": []}])
    new_optimizer.state = optimizer.state
    new_optimizer.param_groups = optimizer.param_groups
    for group in new_optimizer.param_groups:
        for i, p in enumerate(group["params"]):
            if p in param_pair:
                new_param = param_pair[p]
                group["params"][i] = new_param

    return new_model, new_optimizer

    Use TPP to speedup training/inference. fast_bert API is still a prototype
    feature and now only optimized for bert model.

    Args:
        model (torch.nn.Module): User model to apply optimizations on.
        dtype (torch.dtype): Only works for ``torch.bfloat16`` and ``torch.float`` .
            The default value is torch.float.
        optimizer (torch.optim.Optimizer): User optimizer to apply optimizations
            on, such as SGD. The default value is ``None``, meaning inference case.
        unpad(bool): Unpad the squence to reduce the sparsity.
        seed(string): The seed used for the libxsmm kernel. In general it should be same
            to the torch.seed

    .. note::

        Currently ``ipex.fast_bert`` API is well optimized for training tasks.
        It works for inference tasks, though, please use the ``ipex.optimize``
        API with TorchScript to achieve the peak performance.

    .. warning::

        Please invoke ``fast_bert`` function AFTER loading weights to model via
        ``model.load_state_dict(torch.load(PATH))``.

    .. warning::

        This API can't be used when you have applied the ``ipex.optimize``.

    .. warning::

        Please invoke ``optimize`` function BEFORE invoking DDP in distributed
        training scenario.

    Examples:

        >>> # bfloat16 inference case.
        >>> model = ...
        >>> model.load_state_dict(torch.load(PATH))
        >>> model.eval()
        >>> optimized_model = ipex.fast_bert(model, dtype=torch.bfloat16)
        >>> # running evaluation step.
        >>> # bfloat16 training case.
        >>> optimizer = ...
        >>> model.train()
        >>> optimized_model, optimized_optimizer = ipex.fast_bert(model, dtype=torch.bfloat16,
                optimizer=optimizer, unpad=True, seed=args.seed)
        >>> # running training step.

    
intel_extension_for_pytorch/cpu/tpp/fused_bert.py