def _get_num_processes() -> int:
    """Get the number of processes to use for dataset processing."""
    detected_cpu_count = os.cpu_count() or 1
    detected_cpus_with_margin = detected_cpu_count - 8
    num_proc = max(1, detected_cpus_with_margin)

    # Check if the user has set the MAX_NUM_PROC environment variable
    # which caps the number of processes used for dataset processing.
    if 'MAX_NUM_PROC' in os.environ:
        max_num_proc_env = int(os.environ['MAX_NUM_PROC'])
        if max_num_proc_env < num_proc:
            num_proc = max_num_proc_env

    return num_proc
Get the number of processes to use for dataset processing.
llmfoundry/data/finetuning/tasks.py