scripts/benchmark/trainer-benchmark.py/process_run
def process_run(
    id,
    cmd,
    variation_key,
    variation,
    longest_variation_len,
    target_metric_key,
    report_metric_keys,
    repeat_times,
    output_dir,
    verbose,
):
    results = []
    metrics = []
    preamble = f"{id}: {variation:<{longest_variation_len}}"
    outcome = f"{preamble}: "
    metric_keys = set(report_metric_keys + [target_metric_key])
    for i in tqdm(range(repeat_times), desc=preamble, leave=False):
        single_run_metrics = process_run_single(
            id, cmd, variation, output_dir, target_metric_key, metric_keys, verbose
        )
        result = single_run_metrics[target_metric_key]
        if not math.isnan(result):
            metrics.append(single_run_metrics)
            results.append(result)
            outcome += "✓"
        else:
            outcome += "✘"
    outcome = f"\33[2K\r{outcome}"
    if len(metrics) > 0:
        mean_metrics = {k: fmean([x[k] for x in metrics]) for k in metrics[0].keys()}
        mean_target = round(mean_metrics[target_metric_key], 2)
        results_str = f"{outcome} {mean_target}"
        if len(metrics) > 1:
            results_str += f" {tuple(round(x, 2) for x in results)}"
        print(results_str)
        mean_metrics[variation_key] = variation
        return mean_metrics
    else:
        print(outcome)
        return {variation_key: variation, target_metric_key: nan}