def tokenize_example(
        self,
        prompt_and_fewshot: str,
        context_options: list[str],
        example: dict,
    ) -> dict[str, Any]:
        """Runs text through the tokenizer and handle special cases.

        Args:
            prompt_and_fewshot (str): The collection of the prompt and fewshot examples that belongs before the example's context
            context_options (str): A list of contexts for this specific example.
            example (Dict): The example as a dictionary.

        Returns:
            Dict: Dictionary with the tokenized data
        """
        tokenized_example = {}
        preamble = self.tokenizer(prompt_and_fewshot)['input_ids']
        assert isinstance(preamble, list)
        preamble = self._fix_eos_on_preamble(preamble)
        encoded_contexts = [
            preamble
            +  # pyright: ignore[reportOperatorIssue, reportGeneralTypeIssues]
            self.tokenizer(c, add_special_tokens=False)[
                'input_ids']  # pyright: ignore[reportOperatorIssue, ]
            for c in context_options
        ]
        continuation = example['continuation']
        if self.prefix_space:
            continuation = (
                f' {continuation}'
                if not continuation.startswith(' ') else continuation
            )
        tokenized_continuation = self.tokenizer(
            continuation,
            add_special_tokens=False,
        )['input_ids']

        tokenized_example[self.context_key] = []
        tokenized_example['continuation_indices'] = []
        tokenized_example[self.answer_key] = []
        for context in encoded_contexts:
            assert isinstance(context, list)
            assert isinstance(tokenized_continuation, list)
            trimmed_context = trim_context(
                context,
                tokenized_continuation,
                self.padding_size,
            )
            assert isinstance(trimmed_context, list)
            continuation_indices = get_continuation_span(
                trimmed_context,
                tokenized_continuation,
            )
            padded_context = make_padded_input(
                trimmed_context,
                tokenized_continuation,
                self.padding_size,
                self.pad_tok_id,
                self.padding_side,
            )
            tokenized_example[self.context_key].append(padded_context)
            tokenized_example['continuation_indices'].append(
                continuation_indices,
            )
            tokenized_example[self.answer_key].append(tokenized_continuation)

        tokenized_example['gold'] = example['gold']
        return tokenized_example
Runs text through the tokenizer and handle special cases.

        Args:
            prompt_and_fewshot (str): The collection of the prompt and fewshot examples that belongs before the example's context
            context_options (str): A list of contexts for this specific example.
            example (Dict): The example as a dictionary.

        Returns:
            Dict: Dictionary with the tokenized data
        
llmfoundry/eval/datasets/in_context_learning_evaluation.py