def parse_simulation_response(
    response: dict[str, Any],
    prompt_format: PromptFormat,
    tokens: Sequence[str],
) -> SequenceSimulation:
    """
    Parse an API response to a simulation prompt.

    Args:
        response: response from the API
        prompt_format: how the prompt was formatted
        tokens: list of tokens as strings in the sequence where the neuron is being simulated
    """
    choice = response["choices"][0]
    if prompt_format == PromptFormat.CHAT_MESSAGES:
        text = choice["message"]["content"]
    elif prompt_format in [
        PromptFormat.NONE,
        PromptFormat.INSTRUCTION_FOLLOWING,
    ]:
        text = choice["text"]
    else:
        raise ValueError(f"Unhandled prompt format {prompt_format}")
    response_tokens = choice["logprobs"]["tokens"]
    top_logprobs = choice["logprobs"]["top_logprobs"]
    token_text_offset = choice["logprobs"]["text_offset"]
    # This only works because the sequence "<start>" tokenizes into multiple tokens if it appears in
    # a text sequence in the prompt.
    scoring_start = text.rfind("<start>")
    expected_values = []
    original_sequence_tokens: list[str] = []
    distribution_values: list[list[float]] = []
    distribution_probabilities: list[list[float]] = []
    for i in range(2, len(response_tokens)):
        if len(original_sequence_tokens) == len(tokens):
            # Make sure we haven't hit some sort of off-by-one error.
            # TODO(sbills): Generalize this to handle different tokenizers.
            reached_end = response_tokens[i + 1] == "<" and response_tokens[i + 2] == "end"
            assert reached_end, f"{response_tokens[i-3:i+3]}"
            break
        if token_text_offset[i] >= scoring_start:
            # We're looking for the first token after a tab. This token should be the text
            # "unknown" if hide_activations=True or a normalized activation (0-10) otherwise.
            # If it isn't, that means that the tab is not appearing as a delimiter, but rather
            # as a token, in which case we should move on to the next response token.
            if response_tokens[i - 1] == "\t":
                if response_tokens[i] != "unknown":
                    logger.debug("Ignoring tab token that is not followed by an 'unknown' token.")
                    continue

                # j represents the index of the token in a "token<tab>activation" line, barring
                # one of the unusual cases handled below.
                j = i - 2

                current_token = tokens[len(original_sequence_tokens)]
                if current_token == response_tokens[j] or was_token_split(
                    current_token, response_tokens, j
                ):
                    # We're in the normal case where the tokenization didn't throw off the
                    # formatting or in the token-was-split case, which we handle the usual way.
                    current_top_logprobs = top_logprobs[i]

                    (
                        norm_probabilities_by_distribution_value,
                        expected_value,
                    ) = compute_predicted_activation_stats_for_token(
                        current_top_logprobs,
                    )
                    current_distribution_values = list(
                        norm_probabilities_by_distribution_value.keys()
                    )
                    current_distribution_probabilities = list(
                        norm_probabilities_by_distribution_value.values()
                    )
                else:
                    # We're in a case where the tokenization resulted in a newline being folded into
                    # the token. We can't do our usual prediction of activation stats for the token,
                    # since the model did not observe the original token. Instead, we use dummy
                    # values. See the TODO elsewhere in this file about coming up with a better
                    # prompt format that avoids this situation.
                    newline_folded_into_token = "\n" in response_tokens[j]
                    assert (
                        newline_folded_into_token
                    ), f"`{current_token=}` {response_tokens[j-3:j+3]=}"
                    logger.debug(
                        "Warning: newline before a token<tab>activation line was folded into the token"
                    )
                    current_distribution_values = []
                    current_distribution_probabilities = []
                    expected_value = 0.0

                original_sequence_tokens.append(current_token)
                # These values are ints, but for backward compatibility we store them as floats.
                distribution_values.append([float(v) for v in current_distribution_values])
                distribution_probabilities.append(current_distribution_probabilities)
                expected_values.append(expected_value)

    return SequenceSimulation(
        activation_scale=ActivationScale.SIMULATED_NORMALIZED_ACTIVATIONS,
        expected_activations=expected_values,
        distribution_values=distribution_values,
        distribution_probabilities=distribution_probabilities,
        tokens=original_sequence_tokens,
    )

    Parse an API response to a simulation prompt.

    Args:
        response: response from the API
        prompt_format: how the prompt was formatted
        tokens: list of tokens as strings in the sequence where the neuron is being simulated
    
neuron_explainer/explanations/simulator.py