def build_inner_model(
        cls,
        pretrained_model_name_or_path: str,
        pretrained_lora_id_or_path: Optional[str],
        trust_remote_code: bool,
        init_device: str,
        use_flash_attention_2: bool,
        use_auth_token: bool,
        config_overrides: dict[str, Any],
        load_in_8bit: bool,
        pretrained: bool,
        model_cls: Optional[Union[type[_BaseAutoModelClass],
                                  type[PreTrainedModel]]] = None,
        prepare_for_fsdp: bool = False,
        attn_implementation: Optional[str] = None,
    ) -> Union[PreTrainedModel, 'PeftModel']:
        """Builds the inner model for the ComposerHFCausalLM.

        Args:
            pretrained_model_name_or_path (str): The pretrained model name or path.
            pretrained_lora_id_or_path (Optional[str]): The pretrained LORA ID or path.
            trust_remote_code (bool): Whether to trust remote code.
            init_device (str): The initialization device.
            use_flash_attention_2 (bool): Whether to use flash attention 2.
            use_auth_token (bool): Whether to use an authentication token.
            config_overrides (Dict[str, Any]): The configuration overrides.
            load_in_8bit (bool): Whether to load in 8-bit.
            pretrained (bool): Whether the model is pretrained.
            model_cls (Union[Type, Type[PreTrainedModel]]): Kept for backwards compatibility.
            prepare_for_fsdp (bool, optional): Kept for backwards compatilbility.
            attn_implementation (str, optional): The attention implementation to use.
                This will be overridden by if ``use_flash_attention_2`` is ``True``.
                Default: ``None``.

        Returns:
            Union[PreTrainedModel, 'PeftModel']: The built inner model.
        """
        if use_flash_attention_2:
            if attn_implementation is not None and attn_implementation != 'flash_attention_2':
                warnings.warn(
                    'use_flash_attention_2 is set, this will override attn_implementation. '
                    +
                    'Set attn_implementation to flash_attention_2 directly to remove this warning.',
                )
            attn_implementation = 'flash_attention_2'
        if attn_implementation is None:
            attn_implementation = 'eager'

        if pretrained_model_name_or_path.startswith(
            'mosaicml/mpt',
        ):
            raise ValueError(
                'The MPT series of models on the Hugging Face Hub is no longer supported by LLM Foundry. '
                +
                'Please use an older version of LLM Foundry (<0.18) or use a different model. '
                +
                'Please open a GitHub issue if this is a problem for you and we can help you downgrade or work around the issue.',
            )
        # Resolve "mixed" init device to either "cpu" or "meta"
        resolved_init_device = hf_get_init_device(init_device)

        if attn_implementation == 'flash_attention_2' and not is_flash_v2_installed(
        ):
            raise ValueError(
                'use_flash_attention_2 is set to True, but flash-attention 2 is not installed. '
                + 'Please `pip install llm-foundry[gpu]`.',
            )

        auto_model_cls = cls.model_cls if model_cls is None else model_cls

        if not (
            hasattr(auto_model_cls, 'from_pretrained') and
            hasattr(auto_model_cls, 'from_config')
        ):
            raise AttributeError(
                f'{auto_model_cls=} is missing `from_pretrained` and `from_config` support.',
            )

        # Hugging Face copies the modules into the
        # transformers modules cache. On particular systems, this operation seems to cause contention between
        # the different processes. To avoid this contention, we first create the config and generation config on local rank
        # zero. This will set up the transformers module cache and avoid the future contention.
        if dist.get_local_rank() == 0:
            AutoConfig.from_pretrained(
                pretrained_model_name_or_path,
                trust_remote_code=trust_remote_code,
                use_auth_token=use_auth_token,
                attn_implementation=attn_implementation,
                use_cache=
                False,  # Necessary due to https://github.com/huggingface/transformers/issues/28056
            )
            try:
                GenerationConfig.from_pretrained(
                    pretrained_model_name_or_path,
                    use_auth_token=use_auth_token,
                )
            except OSError:
                pass

        dist.barrier()

        # Construct the Hugging Face config to use
        config = cls.build_config(
            pretrained_model_name_or_path,
            trust_remote_code=trust_remote_code,
            use_auth_token=use_auth_token,
            attn_implementation=attn_implementation,
            config_overrides=config_overrides,
        )

        # We need to have all non-zero local ranks be not-pretrained
        # Rank 0 will still be pretrained, and distribute the weights appropriately
        if dist.get_global_rank() != 0 and init_device == 'mixed':
            pretrained = False

        # Hugging Face copies the modules into the
        # transformers modules cache. On particular systems, this operation seems to cause contention between
        # the different processes. To avoid this contention, we first create the model (on meta device) on local rank
        # zero. This will set up the transformers model cache and avoid the future contention.
        if dist.get_local_rank() == 0:
            if pretrained and os.path.isdir(pretrained_model_name_or_path):
                with init_empty_weights(include_buffers=False):
                    with warnings.catch_warnings():
                        warnings.simplefilter('ignore', UserWarning)
                        auto_model_cls.from_pretrained(
                            pretrained_model_name_or_path,
                            trust_remote_code=trust_remote_code,
                            use_auth_token=use_auth_token,
                            attn_implementation=attn_implementation,
                            config=config,
                        )
            else:
                with init_empty_weights(include_buffers=False):
                    auto_model_cls.from_config(  # type: ignore
                        config,
                        trust_remote_code=trust_remote_code,
                        attn_implementation=attn_implementation,
                    )

        dist.barrier()

        def download_thread_target(queue: Optional[queue.Queue]):
            """Thread target to wait for the model to be downloaded."""
            device = get_device(None)

            status = 0

            done_tensor = device.tensor_to_device(
                torch.tensor(
                    [status],
                    dtype=torch.int,
                ),
            )

            dist.broadcast(done_tensor, src=0)
            all_done = done_tensor.item()

            while not all_done:
                # On rank 0, queue exists, and the presence of an item indicates completion
                # On all other ranks, the queue object does not exist
                if queue is not None and not queue.empty():
                    status = 1
                done_tensor.fill_(status)
                dist.broadcast(done_tensor, src=0)
                all_done = done_tensor.item()
                time.sleep(1)

        # Create a thread to busy wait for the download to be complete
        # This approach is used because NCCL process group does not support updating the timeout
        # and we don't want to require a super long distributed timeout for all operations
        wait_for_download_thread = None
        download_status_queue = None
        if dist.get_world_size() > 1:
            download_status_queue = queue.Queue() if dist.get_global_rank(
            ) == 0 else None

            # Create a thread to busy wait for download to be complete
            wait_for_download_thread = threading.Thread(
                target=download_thread_target,
                daemon=True,
                args=(download_status_queue,),
            )
            log.debug('Starting download thread')
            wait_for_download_thread.start()

        model = None
        error = None
        try:
            # initialize the model on the correct device
            if resolved_init_device == 'cpu':
                if pretrained:
                    model = auto_model_cls.from_pretrained(
                        pretrained_model_name_or_path,
                        trust_remote_code=trust_remote_code,
                        use_auth_token=use_auth_token,
                        load_in_8bit=load_in_8bit,
                        attn_implementation=attn_implementation,
                        config=config,
                    )
                else:
                    model = auto_model_cls.from_config(  # type: ignore
                        config,
                        trust_remote_code=trust_remote_code,
                        attn_implementation=attn_implementation,
                    )
            elif resolved_init_device == 'meta':
                if pretrained:
                    raise ValueError(
                        'Setting cfg.pretrained=True is not supported when init_device="meta".',
                    )
                with init_empty_weights(include_buffers=False):
                    model = auto_model_cls.from_config(  # type: ignore
                        config,
                        trust_remote_code=trust_remote_code,
                        attn_implementation=attn_implementation,
                    )
            else:
                raise ValueError(
                    f'Got {init_device=} which resolved to unknown device {resolved_init_device=} on global rank {dist.get_global_rank()}.',
                )
        except Exception as e:
            error = e
            log.error(e)

        # Signal that the download is complete on rank 0
        if download_status_queue is not None:
            download_status_queue.put(1)
            log.debug('Download complete')

        # Join and wait for the thread to complete on all ranks
        if wait_for_download_thread is not None:
            log.debug('Joining download thread')
            wait_for_download_thread.join(timeout=3600)
            log.debug('Download thread joined')

        # Gather information about which ranks errored
        gathered_errors = dist.all_gather_object(error)
        ranks_with_error = [
            rank for rank, err in enumerate(gathered_errors) if err is not None
        ]
        if len(ranks_with_error) > 0:
            raise RuntimeError(
                f'Error initializing model on ranks {ranks_with_error}. See individual rank logs for more details',
            )

        assert model is not None
        # Use the pretrained generation config for the model if it exists.
        try:
            model.generation_config = GenerationConfig.from_pretrained(
                pretrained_model_name_or_path,
                use_auth_token=use_auth_token,
            )
        except OSError:
            log.warning(
                f'No existing generation config found for the model with name or path {pretrained_model_name_or_path}. Using default generation config.',
            )

        # Hugging Face's weight tying does not succeed if the model is inited on meta device
        # so we manually apply the weight tying here
        if model.config.tie_word_embeddings and resolved_init_device == 'meta':
            model.tie_weights()

        if pretrained_lora_id_or_path is not None:
            if not peft_installed:
                raise ValueError(
                    'PEFT is not installed, but lora_id_or_path was passed. Please install LLM Foundry with the peft extra to use lora_id_or_path.',
                )
            from peft import PeftModelForCausalLM
            model = PeftModelForCausalLM.from_pretrained(
                model,
                pretrained_lora_id_or_path,
                is_trainable=True,
            )

        if prepare_for_fsdp:
            cls.prepare_inner_model(model, init_device)

        return model
Builds the inner model for the ComposerHFCausalLM.

        Args:
            pretrained_model_name_or_path (str): The pretrained model name or path.
            pretrained_lora_id_or_path (Optional[str]): The pretrained LORA ID or path.
            trust_remote_code (bool): Whether to trust remote code.
            init_device (str): The initialization device.
            use_flash_attention_2 (bool): Whether to use flash attention 2.
            use_auth_token (bool): Whether to use an authentication token.
            config_overrides (Dict[str, Any]): The configuration overrides.
            load_in_8bit (bool): Whether to load in 8-bit.
            pretrained (bool): Whether the model is pretrained.
            model_cls (Union[Type, Type[PreTrainedModel]]): Kept for backwards compatibility.
            prepare_for_fsdp (bool, optional): Kept for backwards compatilbility.
            attn_implementation (str, optional): The attention implementation to use.
                This will be overridden by if ``use_flash_attention_2`` is ``True``.
                Default: ``None``.

        Returns:
            Union[PreTrainedModel, 'PeftModel']: The built inner model.
        
llmfoundry/models/hf/hf_base.py