def evaluate(
    dataset,
    vectors_path,
    strategy="most_similar",
    senses=None,
    exclude_senses=EVAL_EXCLUDE_SENSES,
    n_freq=100_000,
    threshold=0.7,
    batch_size=10,
    eval_whole=False,
    eval_only=False,
    show_scores=False,
):
    """
    Evaluate a sense2vec model by asking about phrase triples: is word A more
    similar to word B, or to word C? If the human mostly agrees with the model,
    the vectors model is good.
    """
    random.seed(0)
    log("RECIPE: Starting recipe sense2vec.eval", locals())
    strategies = eval_strategies.get_all()
    if strategy not in strategies.keys():
        err = f"Invalid strategy '{strategy}'. Expected: {list(strategies.keys())}"
        msg.fail(err, exits=1)
    s2v = Sense2Vec().from_disk(vectors_path)
    log("RECIPE: Loaded sense2vec vectors", vectors_path)

    def get_html(key, score=None, large=False):
        word, sense = s2v.split_key(key)
        html_word = f"<span style='font-size: {30 if large else 20}px'>{word}</span>"
        html_sense = f"<strong style='opacity: 0.75; font-size: 14px; padding-left: 10px'>{sense}</strong>"
        html = f"{html_word} {html_sense}"
        if show_scores and score is not None:
            html += f" <span style='opacity: 0.75; font-size: 12px; padding-left: 10px'>{score:.4}</span>"
        return html

    def get_stream():
        strategy_func = eval_strategies.get(strategy)
        log(f"RECIPE: Using strategy {strategy}")
        # Limit to most frequent entries
        keys = [key for key, _ in s2v.frequencies[:n_freq]]
        keys_by_sense = defaultdict(set)
        for key in keys:
            try:
                sense = s2v.split_key(key)[1]
            except ValueError:
                continue
            if (senses is None or sense in senses) and sense not in exclude_senses:
                keys_by_sense[sense].add(key)
        keys_by_sense = {s: keys for s, keys in keys_by_sense.items() if len(keys) >= 3}
        all_senses = list(keys_by_sense.keys())
        total_keys = sum(len(keys) for keys in keys_by_sense.values())
        log(f"RECIPE: Using {total_keys} entries for {len(all_senses)} senses")
        n_passes = 1
        while True:
            log(f"RECIPE: Iterating over the data ({n_passes})")
            current_keys = copy.deepcopy(keys_by_sense)
            while any(len(values) >= 3 for values in current_keys.values()):
                sense = random.choice(all_senses)
                all_keys = list(current_keys[sense])
                key_a, key_b, key_c, sim_ab, sim_ac = strategy_func(s2v, all_keys)
                if len(set([key_a.lower(), key_b.lower(), key_c.lower()])) != 3:
                    continue
                if sim_ab < threshold or sim_ac < threshold:
                    continue
                for key in (key_a, key_b, key_c):
                    current_keys[sense].remove(key)
                confidence = 1.0 - (min(sim_ab, sim_ac) / max(sim_ab, sim_ac))
                input_hash = murmurhash.hash(key_a)
                task_hash = murmurhash.hash(" ".join([key_a] + sorted([key_b, key_c])))
                task = {
                    "label": "Which one is more similar?",
                    "html": get_html(key_a, large=True),
                    "text": f"{key_a}: {key_b}, {key_c}",
                    "key": key_a,
                    "options": [
                        {
                            "id": key_b,
                            "html": get_html(key_b, sim_ab),
                            "score": sim_ab,
                        },
                        {
                            "id": key_c,
                            "html": get_html(key_c, sim_ac),
                            "score": sim_ac,
                        },
                    ],
                    "confidence": confidence,
                    TASK_HASH_ATTR: task_hash,
                    INPUT_HASH_ATTR: input_hash,
                }
                if show_scores:
                    task["meta"] = {
                        "confidence": f"{confidence:.4}",
                        "strategy": strategy,
                    }
                yield task
            n_passes += 1

    def eval_dataset(set_id):
        """Output summary about user agreement with the model."""
        DB = connect()
        data = DB.get_dataset(set_id)
        accepted = [eg for eg in data if eg["answer"] == "accept" and eg.get("accept")]
        rejected = [eg for eg in data if eg["answer"] == "reject"]
        if not accepted and not rejected:
            msg.warn("No annotations collected", exits=1)
        high_conf = 0.8
        agree_count = 0
        disagree_high_conf = len([e for e in rejected if e["confidence"] > high_conf])
        for eg in accepted:
            choice = eg["accept"][0]
            score_choice = [o["score"] for o in eg["options"] if o["id"] == choice][0]
            score_other = [o["score"] for o in eg["options"] if o["id"] != choice][0]
            if score_choice > score_other:
                agree_count += 1
            elif eg["confidence"] > high_conf:
                disagree_high_conf += 1
        pc = agree_count / (len(accepted) + len(rejected))
        text = f"You agreed {agree_count} / {len(data)} times ({pc:.0%})"
        msg.info(f"Evaluating data from '{set_id}'")
        if pc > 0.5:
            msg.good(text)
        else:
            msg.fail(text)
        msg.text(f"You disagreed on {disagree_high_conf} high confidence scores")
        msg.text(f"You rejected {len(rejected)} suggestions as not similar")

    def on_exit(ctrl):
        set_id = dataset if eval_whole else ctrl.session_id
        eval_dataset(set_id)

    if eval_only:
        eval_dataset(dataset)
        return None

    return {
        "view_id": "choice",
        "dataset": dataset,
        "stream": get_stream(),
        "on_exit": on_exit,
        "config": {
            "batch_size": batch_size,
            "choice_style": "single",
            "choice_auto_accept": True,
        },
    }

    Evaluate a sense2vec model by asking about phrase triples: is word A more
    similar to word B, or to word C? If the human mostly agrees with the model,
    the vectors model is good.
    
sense2vec/prodigy_recipes.py