def _global_except_hook(exctype, value, traceback):
    """Catches an unhandled exception and call MPI_Abort()."""
    try:
        if _orig_except_hook:
            _orig_except_hook(exctype, value, traceback)
        else:
            sys.__excepthook__(exctype, value, traceback)

    finally:
        import mpi4py.MPI
        rank = mpi4py.MPI.COMM_WORLD.Get_rank()
        logger.warning("******************************************")
        logger.warning("DefaultTrainer:")
        logger.warning(f"   Uncaught exception on rank {rank}.")
        logger.warning("   Calling MPI_Abort() to shut down MPI...")
        logger.warning("******************************************")
        logging.shutdown()

        try:
            import mpi4py.MPI
            mpi4py.MPI.COMM_WORLD.Abort(1)
        except Exception as e:
            # Something is completely broken...
            # There's nothing we can do any more
            sys.stderr.write("Sorry, failed to stop MPI and the process may hang.\n")
            sys.stderr.flush()
            raise e
Catches an unhandled exception and call MPI_Abort().
trainer/utils/hook.py