async def score_explanation(
        self,
        explanation: str,
        activation_records: list[ActivationRecord],
        max_activation: float,
        # The number of high and low activating token pairs to sample for simulation
        num_activations_for_scoring: int = 5,
        # The activation threshold below which a token pair is eligible for sampling
        # as a low activating pair.
        low_activation_threshold: float = 0.1,
    ) -> ScoredAttentionSimulation:
        """Score explanations based on how well they predict attention between
        top attending token pairs and random low attending token pairs."""
        # Use the activation records to generate a set of pairs for scoring.
        # 10 pairs: the five top activating pairs, and five randomly chosen pairs
        # where the activations are below 0.1 * the max value.
        candidates = []
        for i, record in enumerate(activation_records):
            sorted_activation_flat_indices = np.argsort(record.activations)[::-1]
            sorted_vals = [record.activations[idx] for idx in sorted_activation_flat_indices]
            coordinates = [
                convert_flattened_index_to_unflattened_index(flat_index)
                for flat_index in sorted_activation_flat_indices
            ]
            candidates.extend([(i, val, coords) for val, coords in zip(sorted_vals, coordinates)])
        top_activation_coordinates = [
            (candidate[0], candidate[2])
            for candidate in sorted(candidates, key=lambda x: x[1], reverse=True)
        ][:num_activations_for_scoring]

        filtered_low_activation_coordinates = [
            (candidate[0], candidate[2])
            for candidate in candidates
            if candidate[1] < low_activation_threshold * max_activation
        ]
        selected_low_activation_coordinates = random.sample(
            filtered_low_activation_coordinates,
            min(len(filtered_low_activation_coordinates), num_activations_for_scoring),
        )

        attention_simulations = []
        true_labels = [1 for _ in range(len(top_activation_coordinates))] + [
            0 for _ in range(len(selected_low_activation_coordinates))
        ]
        # No need to shuffle because the model only sees one at a time anyway.
        for coords, label in zip(
            top_activation_coordinates + selected_low_activation_coordinates, true_labels
        ):
            activation_record = activation_records[coords[0]]
            # for each pair, generate a prompt where the model is asked to predict if the token pair has a strong
            # or weak activation.
            prompt = self.make_token_pair_prompt(explanation, activation_record.tokens, coords[1])

            assert isinstance(prompt, list)
            assert isinstance(prompt[0], dict)  # Really a ChatMessage
            generate_kwargs: dict[str, Any] = {
                # Using a timeout prevents the scorer from hanging if the API server is overloaded.
                "timeout": 60,
                "n": 1,
                "max_tokens": 1,  # we only want to sample one token.
                "logprobs": True,
                "top_logprobs": 15,
                "messages": prompt,
            }
            response = await self.client.async_generate(**generate_kwargs)
            assert len(response["choices"]) == 1

            # from the response, extract the logit values for "0" (for weak) and "1" (for strong) to obtain
            # a float.
            choice = response["choices"][0]
            # for whatever reason `choice["logprobs"]["top_logprobs"]` is a list of dicts
            logprobs_dicts = choice["logprobs"]["content"][0]["top_logprobs"]
            extracted_probs = {d["token"]: d["logprob"] for d in logprobs_dicts}
            zero_prob = np.exp(extracted_probs["0"]) if "0" in extracted_probs else 0.0
            one_prob = np.exp(extracted_probs["1"]) if "1" in extracted_probs else 0.0
            total_prob = zero_prob + one_prob
            # The score is 0 * normalized probability of "0" + 1 * normalized probability of "1", which
            # reduces to just the normalized probability of "1".
            normalized_one_prob = one_prob / total_prob
            # print(f"zero_prob: {zero_prob/total_prob}, one_prob: {normalized_one_prob}")
            attention_simulations.append(
                AttentionSimulation(
                    tokens=activation_record.tokens,
                    token_pair_coords=coords[1],
                    token_pair_label=label,
                    simulation_prediction=normalized_one_prob,
                )
            )

        assert (
            len(attention_simulations)
            == len(true_labels)
            == len(top_activation_coordinates) + len(selected_low_activation_coordinates)
        )

        # ROC AUC awards a perfect score to explanations that order all of the scores
        # for pairs labeled "1" above the scores for pairs labeled "0" (even if the scores
        # for pairs labeled "0" are well above 0).
        score = roc_auc_score(
            y_true=true_labels, y_score=[sim.simulation_prediction for sim in attention_simulations]
        )
        return ScoredAttentionSimulation(
            attention_simulations=attention_simulations,
            roc_auc_score=score,
        )
Score explanations based on how well they predict attention between
        top attending token pairs and random low attending token pairs.
neuron_explainer/explanations/attention_head_scoring.py