def _tokenize_with_bos_removal(
    tokenizer: PreTrainedTokenizerBase,
    text: str,
    text_target: str,
) -> dict[str, list[int]]:
    """Tokenizes the prompt and response using the provided tokenizer.

    Args:
        tokenizer (PreTrainedTokenizerBase): The tokenizer to use for tokenization.
        text (str): The prompt to tokenize.
        text_target (str): The response to tokenize.

    Returns:
        Dict[str, List[int]]: The tokenized text and text_target.
    """
    tokenized_sample = tokenizer(
        text=text,
        text_target=text_target,
        padding=False,
        truncation=False,
    )

    # Remove the BOS token from the start of the labels if it was automatically added
    # Unfortunately if the tokenizer is PretrainedTokenizerFast, as llama3 is, it does not provide
    # an add_bos_token attr that we can check explicitly, so instead we rely on checking if both the
    # text and the text_target start with bos_token_id to determine whether to strip bos.
    has_bos_token = hasattr(
        tokenizer,
        'bos_token_id',
    ) and tokenizer.bos_token_id is not None
    input_ids_starts_with_bos = False
    labels_starts_with_bos = False
    if has_bos_token and len(
        cast(Any, tokenized_sample['input_ids']),
    ) > 0 and len(cast(Any, tokenized_sample['labels'])) > 0:
        input_ids_starts_with_bos = cast(
            Any,
            tokenized_sample['input_ids'],
        )[0] == tokenizer.bos_token_id
        labels_starts_with_bos = cast(
            Any,
            tokenized_sample['labels'],
        )[0] == tokenizer.bos_token_id
    if input_ids_starts_with_bos and labels_starts_with_bos:
        tokenized_sample['labels'] = cast(Any, tokenized_sample['labels'])[1:]

    return tokenized_sample  # type: ignore
Tokenizes the prompt and response using the provided tokenizer.

    Args:
        tokenizer (PreTrainedTokenizerBase): The tokenizer to use for tokenization.
        text (str): The prompt to tokenize.
        text_target (str): The response to tokenize.

    Returns:
        Dict[str, List[int]]: The tokenized text and text_target.
    
llmfoundry/data/finetuning/tasks.py