def eval_ab(
    dataset,
    vectors_path_a,
    vectors_path_b,
    senses=None,
    exclude_senses=EVAL_EXCLUDE_SENSES,
    n_freq=100_000,
    n_similar=10,
    batch_size=5,
    eval_whole=False,
    eval_only=False,
    show_mapping=False,
):
    """
    Perform an A/B evaluation of two pretrained sense2vec vector models by
    comparing the most similar entries they return for a random phrase. The
    UI shows two randomized options with the most similar entries of each model
    and highlights the phrases that differ. At the end of the annotation
    session the overall stats and preferred model are shown.
    """
    log("RECIPE: Starting recipe sense2vec.eval-ab", locals())
    random.seed(0)
    s2v_a = Sense2Vec().from_disk(vectors_path_a)
    s2v_b = Sense2Vec().from_disk(vectors_path_b)
    mapping = {"A": vectors_path_a, "B": vectors_path_b}
    log("RECIPE: Loaded sense2vec vectors", (vectors_path_a, vectors_path_b))
    seen = set()
    DB = connect()
    if dataset in DB:
        examples = DB.get_dataset(dataset)
        seen.update([eg["text"] for eg in examples if eg["answer"] == "accept"])
        log(f"RECIPE: Skipping {len(seen)} terms already in dataset")

    def get_term_html(key):
        word, sense = s2v_a.split_key(key)
        return (
            f"<span style='font-size: 30px'>{word}</span> "
            f"<strong style='opacity: 0.75; font-size: 14px; padding-left: "
            f"10px'>{sense}</strong>"
        )

    def get_option_html(most_similar, overlap):
        html = []
        for key in most_similar:
            font_weight = "normal" if key in overlap else "bold"
            border_color = "#f6f6f6" if key in overlap else "#ccc"
            word, sense = s2v_a.split_key(key)
            html.append(
                f"<span style='display: inline-block; background: #f6f6f6; "
                f"font-weight: {font_weight}; border: 1px solid {border_color}; "
                f"padding: 0 8px; margin: 0 5px 5px 0; border-radius: 5px; "
                f"white-space: nowrap'>{word} <span style='font-weight: bold; "
                f"text-transform: uppercase; font-size: 10px; margin-left: "
                f"5px'>{sense}</span></span>"
            )
        html = " ".join(html) if html else "<em>No results</em>"
        return (
            f"<div style='font-size: 16px; line-height: 1.75; padding: 5px "
            f"12px 5px 0'>{html}</div>"
        )

    def get_stream():
        keys_a = [key for key, _ in s2v_a.frequencies[:n_freq] if key not in seen]
        keys_b = [key for key, _ in s2v_b.frequencies[:n_freq] if key not in seen]
        while len(keys_a):
            key = random.choice(keys_a)
            keys_a.remove(key)
            word, sense = s2v_a.split_key(key)
            if sense in exclude_senses or (senses is not None and sense not in senses):
                continue
            if key not in keys_b:
                continue
            similar_a = set([k for k, _ in s2v_a.most_similar(key, n=n_similar)])
            similar_b = set([k for k, _ in s2v_b.most_similar(key, n=n_similar)])
            overlap = similar_a.intersection(similar_b)
            options = [
                {"id": "A", "html": get_option_html(similar_a, overlap)},
                {"id": "B", "html": get_option_html(similar_b, overlap)},
            ]
            random.shuffle(options)
            task_hash = murmurhash.hash(key)
            task = {
                "html": get_term_html(key),
                "text": key,
                "options": options,
                TASK_HASH_ATTR: task_hash,
                INPUT_HASH_ATTR: task_hash,
            }
            if show_mapping:
                opt_map = [f"{opt['id']} ({mapping[opt['id']]})" for opt in options]
                task["meta"] = {i + 1: opt for i, opt in enumerate(opt_map)}
            yield task

    def eval_dataset(set_id):
        DB = connect()
        data = DB.get_dataset(set_id)
        accepted = [eg for eg in data if eg["answer"] == "accept" and eg.get("accept")]
        rejected = [eg for eg in data if eg["answer"] == "reject"]
        ignored = [eg for eg in data if eg["answer"] == "ignore"]
        if not accepted and not rejected:
            msg.warn("No annotations collected", exits=1)
        counts = Counter()
        for eg in accepted:
            for model_id in eg["accept"]:
                counts[model_id] += 1
        preference, _ = counts.most_common(1)[0]
        ratio = f"{counts[preference]} / {sum(counts.values()) - counts[preference]}"
        msg.info(f"Evaluating data from '{set_id}'")
        msg.text(f"You rejected {len(rejected)} and ignored {len(ignored)} pair(s)")
        if counts["A"] == counts["B"]:
            msg.warn(f"No preference ({ratio})")
        else:
            pc = counts[preference] / sum(counts.values())
            msg.good(f"You preferred vectors {preference} with {ratio} ({pc:.0%})")
            msg.text(mapping[preference])

    def on_exit(ctrl):
        set_id = dataset if eval_whole else ctrl.session_id
        eval_dataset(set_id)

    if eval_only:
        eval_dataset(dataset)
        return None

    return {
        "view_id": "choice",
        "dataset": dataset,
        "stream": get_stream(),
        "on_exit": on_exit,
        "config": {
            "batch_size": batch_size,
            "choice_style": "single",
            "choice_auto_accept": True,
        },
    }

    Perform an A/B evaluation of two pretrained sense2vec vector models by
    comparing the most similar entries they return for a random phrase. The
    UI shows two randomized options with the most similar entries of each model
    and highlights the phrases that differ. At the end of the annotation
    session the overall stats and preferred model are shown.
    
sense2vec/prodigy_recipes.py