scripts/benchmark/trainer-benchmark.py/process_run_single
def process_run_single(id, cmd, variation, output_dir, target_metric_key, metric_keys, verbose):

    # Enable to debug everything but the run itself, to do it fast and see the progress.
    # This is useful for debugging the output formatting quickly - we can remove it later once
    # everybody is happy with the output
    if 0:
        import random
        from time import sleep

        sleep(0)
        return dict(
            {k: random.uniform(0, 100) for k in metric_keys},
            **{target_metric_key: random.choice([nan, 10.31, 100.2, 55.6666, 222.22222222])},
        )

    result = subprocess.run(cmd, capture_output=True, text=True)

    if verbose:
        print("STDOUT", result.stdout)
        print("STDERR", result.stderr)

    # save the streams
    prefix = variation.replace(" ", "-")
    with open(Path(output_dir) / f"log.{prefix}.stdout.txt", "w") as f:
        f.write(result.stdout)
    with open(Path(output_dir) / f"log.{prefix}.stderr.txt", "w") as f:
        f.write(result.stderr)

    if result.returncode != 0:
        if verbose:
            print("failed")
        return {target_metric_key: nan}

    with io.open(f"{output_dir}/all_results.json", "r", encoding="utf-8") as f:
        metrics = json.load(f)

    # filter out just the keys we want
    return {k: v for k, v in metrics.items() if k in metric_keys}