def rotate_model(
        self,
        model: HfModelHandler,
        rotate_mode: str,
        seed: int,
        training_args: Optional[BaseHFTrainingArguments] = None,
    ):
        """Create a new model with rotate modules.

        :param model: HfModelHandler: The model to rotate.
        :param rotate_mode: str: The rotation method to use.
        :param seed: int: The random seed for the rotation.
        :param training_args: Optional[BaseHFTrainingArguments]: The training arguments for the model.
        :return: ModelWrapper with the rotated model, rotation parameters, and save replacements.
        """
        if model.adapter_path:
            logger.info(
                "Model has adapters but %s does not support adapters. Rotating without adapters. The original"
                " adapters will be used as is with the rotated base model.",
                self.__class__.__name__,
            )

            # create a new input model with the adapter path removed
            model.model = None
            model = deepcopy(model)
            model.set_resource("adapter_path", None)

        # load pytorch model
        # use "auto" by default to keep the original dtype
        torch_dtype = model.get_load_kwargs().get("torch_dtype") or "auto"
        if training_args:
            torch_dtype = torch.bfloat16 if training_args.bf16 else torch.float16
        pytorch_model = load_hf_base_model(model, torch_dtype=torch_dtype)

        # prepare model for finetuning
        if training_args:
            prepare_model_for_finetuning(pytorch_model, training_args)

        # create model model_wrapper
        model_wrapper = ModelWrapper.from_model(pytorch_model)

        # fuse layernorms into adjacent linear layers
        self.fuse_layer_norms(model_wrapper)

        # rotate the model
        torch.manual_seed(seed)
        rotation_params = []
        R1 = torch.nn.Parameter(
            self.get_orthogonal_matrix(model_wrapper.hidden_size, rotate_mode, model_wrapper.model.device)
        )
        rotation_params.append(R1)

        # rotate embeddings and lm_head
        for embed, embed_name in zip(*model_wrapper.get_embeds()):
            # embed_tokens @ R1
            set_attr(model_wrapper.model, embed_name, RotateEmbed(embed, R1))
        # R1^-1 @ Whead
        lm_head, lm_head_name = model_wrapper.get_lm_head()
        set_attr(model_wrapper.model, lm_head_name, RotateLinear(lm_head, Q_pre=R1))

        # need v_proj to be rotated separately, so unpack if necessary
        model_wrapper.maybe_unpack_qkv()

        # rotate the hidden layers
        for layer_wrapper in model_wrapper.get_layer_wrappers():
            R2 = None
            for linear_idx, (linear, linear_name) in enumerate(zip(*layer_wrapper.get_attention_inputs())):
                # R1^-1 @ Wq, R1^-1 @ Wk, R1^-1 @ Wv @ R2
                if linear_idx == 2 and getattr(linear, "bias", None) is None:
                    # original implementation ignores bias but output doesn't match both when bias is
                    # rotated headwise and when it is not, so we skip it for now
                    # not really an issue since bias is not present in most models
                    R2 = torch.nn.Parameter(
                        self.get_orthogonal_matrix(model_wrapper.head_dim, rotate_mode, model_wrapper.model.device)
                    )
                    rotation_params.append(R2)
                set_attr(
                    layer_wrapper.layer,
                    linear_name,
                    RotateLinear(linear, Q_pre=R1, Q_post=R2 if linear_idx == 2 else None),
                )

            for linear, linear_name in zip(*layer_wrapper.get_attention_outputs()):
                # R2^-1 @ Wo @ R1
                set_attr(layer_wrapper.layer, linear_name, RotateLinear(linear, Q_pre=R2, Q_post=R1))

            for linear, linear_name in zip(*layer_wrapper.get_mlp_inputs()):
                # R1^-1 @ Wup, R1^-1 @ Wgate
                set_attr(layer_wrapper.layer, linear_name, RotateLinear(linear, Q_pre=R1))

            for linear, linear_name in zip(*layer_wrapper.get_mlp_outputs()):
                # Wdown @ R1
                set_attr(layer_wrapper.layer, linear_name, RotateLinear(linear, Q_post=R1))

        if training_args:
            logger.debug(
                "The number of trainable parameters in the trainable model: %s",
                count_trainable_parameters(model_wrapper.model),
            )

        return (
            model_wrapper,
            rotation_params,
            [((RotateEmbed, RotateLinear), lambda x: x.create_merged("cuda" if torch.cuda.is_available() else "cpu"))],
        )
Create a new model with rotate modules.

        :param model: HfModelHandler: The model to rotate.
        :param rotate_mode: str: The rotation method to use.
        :param seed: int: The random seed for the rotation.
        :param training_args: Optional[BaseHFTrainingArguments]: The training arguments for the model.
        :return: ModelWrapper with the rotated model, rotation parameters, and save replacements.
        
olive/passes/pytorch/rotate.py