def monitor_and_plot(prefix="result/tmp", display_log=False, stop_flag: Value = None):
    """Monitor and plot GPU usage.
    Args:
        prefix: The prefix of the output file.
        stop_flag: A multiprocessing.Value to indicate if monitoring should stop.
    """
    devices = Device.all()
    initial_pids = set()
    monitored_pids = set()

    with open(f"{prefix}.csv", "w", newline="") as csvfile:
        csvwriter = csv.writer(csvfile)
        csvwriter.writerow(["Time", "Used Memory%", "GPU Utilization"])

        try:
            while True:
                if stop_flag and stop_flag.value:
                    break

                for device in devices:
                    current_pids = set(device.processes().keys())
                    if not initial_pids:
                        initial_pids = current_pids

                    new_pids = current_pids - initial_pids
                    if new_pids:
                        monitored_pids.update(new_pids)

                    for pid in monitored_pids.copy():
                        if pid not in current_pids:
                            monitored_pids.remove(pid)
                            if not monitored_pids:
                                raise StopIteration
                    log_device_info(device, prefix, csvwriter, display_log)
                time.sleep(1)
        except StopIteration:
            pass

    plot_data(prefix)
    return
Monitor and plot GPU usage.
    Args:
        prefix: The prefix of the output file.
        stop_flag: A multiprocessing.Value to indicate if monitoring should stop.
    
scripts/train_kohya/utils/gpu_info.py