def sync_params(module):
    """ Sync all parameters of module across all MPI processes. """
    if num_procs()==1:
        return
    for p in module.parameters():
        p_numpy = p.data.numpy()
        broadcast(p_numpy)
 Sync all parameters of module across all MPI processes. 
spinup/utils/mpi_pytorch.py