def evaluate_checkpoints(
    estimator,
    caption_input_fn=None,
    caption_eval_fn=None,
    vqa_input_fn=None,
    vqa_eval_fn=None,
    max_checkpoint_number=None,
):
  """Continuously loops while checking for checkpoints to evaluate.

  Args:
    estimator: Instance of Estimator.
    caption_input_fn: Estimator input function for supervised captioning.
    caption_eval_fn: Evaluation callback on captioning output.
    vqa_input_fn: Estimator input function for visual qa.
    vqa_eval_fn: Evaluation callback on visual qa output.
    max_checkpoint_number: Stop after this checkpoint is evaluated.
  """

  def _compare_metrics(x, y):
    """Compare metrics of result x to result y."""
    metric_x = FLAGS.metric_sign * x.get(FLAGS.metric, 0)
    metric_y = FLAGS.metric_sign * y.get(FLAGS.metric, 0)
    tf.logging.info("Metric: %s\tNew: %2.4f\tOld: %2.4f", FLAGS.metric,
                    metric_x, metric_y)
    return metric_x > metric_y

  # Store best
  best_results = None
  best_path = os.path.join(estimator.model_dir, "model.ckpt-best")

  # Init async threads.
  threads = multiprocessing.pool.ThreadPool(2)

  # Reload best from disk.
  if tf.io.gfile.exists(best_path + ".eval"):
    with tf.io.gfile.GFile(best_path + ".eval", "r") as f:
      best_results = json.load(f)

  # Summary writer
  summaries = tf.summary.FileWriter(os.path.join(estimator.model_dir, "eval"))

  # Track evaluated checkpoints.
  ckpt_manager = os.path.join(estimator.model_dir, "evaluated_ckpts.json")
  tf.io.gfile.makedirs(estimator.model_dir)
  if tf.io.gfile.exists(ckpt_manager):
    with tf.io.gfile.GFile(ckpt_manager, "r") as f:
      visited = set(json.load(f))
  else:
    visited = set()

  # Iterate over checkpoint files as they appear.
  for ckpt in checkpoints_iterator(estimator.model_dir):
    current_step = int(os.path.basename(ckpt).split("-")[1])

    # Skip if already evaluated.
    if ckpt in visited:
      continue

    tf.logging.info("***** Starting evaluation *****")
    try:
      # Initialize resuls.
      results = dict(step=current_step)

      # Evaluate predictions concurrently.
      async_results = []

      # Generate captions.
      if caption_input_fn is not None:
        tf.logging.info("Writing captions predictions to disk...")
        with tf.io.gfile.GFile(ckpt + ".caption_pred", "w") as f:
          iterator = estimator.predict(
              input_fn=functools.partial(
                  caption_input_fn, mode=tf_estimator.ModeKeys.PREDICT),
              checkpoint_path=ckpt,
              yield_single_examples=True)
          i = 0
          for ex in iterator:
            i += 1
            f.write(json.dumps(ex, cls=io_utils.NumpyEncoder) + "\n")
            if i % 1000 == 0:
              tf.logging.info("Wrote %d predictions", i)
          tf.logging.info("Done. Wrote %d predictions.", i)

      # Evaluate captions.
      if caption_eval_fn is not None:
        async_results.append(
            threads.apply_async(caption_eval_fn, (ckpt + ".caption_pred",)))

      # Generate predictions for questions.
      if vqa_input_fn is not None:
        tf.logging.info("Writing question predictions to disk...")
        with tf.io.gfile.GFile(ckpt + ".question_pred", "w") as f:
          iterator = estimator.predict(
              input_fn=functools.partial(
                  vqa_input_fn, mode=tf_estimator.ModeKeys.PREDICT),
              checkpoint_path=ckpt,
              yield_single_examples=True)
          i = 0
          for ex in iterator:
            i += 1
            # Take top-1 caption.
            ex["token_ids"] = ex["token_ids"][:1]
            f.write(json.dumps(ex, cls=io_utils.NumpyEncoder) + "\n")
            if i % 1000 == 0:
              tf.logging.info("Wrote %d predictions", i)
          tf.logging.info("Done. Wrote %d predictions.", i)

      # Evaluate questions.
      if vqa_eval_fn is not None:
        async_results.append(
            threads.apply_async(vqa_eval_fn, (ckpt + ".question_pred",)))

      # Wait for results.
      for res in async_results:
        results.update(res.get())
      tf.logging.info(results)

      # Log all results.
      for k, v in results.items():
        summary = tf.Summary(value=[tf.Summary.Value(tag=k, simple_value=v)])
        summaries.add_summary(summary, current_step)

      # Save results.
      with tf.io.gfile.GFile(ckpt + ".eval", "w") as f:
        json.dump(
            results, f, cls=io_utils.NumpyEncoder, indent=2, sort_keys=True)

      # Mark checkpoint.
      visited.add(ckpt)
      with tf.io.gfile.GFile(ckpt_manager, "w") as f:
        json.dump(list(visited), f)

      # Maybe update best copy.
      if best_results is None or _compare_metrics(results, best_results):
        tf.logging.info("New best evaluation.")
        best_results = results
        for fname in tf.io.gfile.glob(ckpt + "*"):
          ext = os.path.splitext(fname)[-1]
          tf.logging.info("Copying %s to best...", fname)
          tf.io.gfile.copy(fname, best_path + ext, overwrite=True)

      is_final = max_checkpoint_number and current_step >= max_checkpoint_number
      if is_final:
        tf.logging.info("Evaluation finished after checkpoint step number %d.",
                        current_step)
        tf.logging.info("Terminating evaluation.")
        summaries.close()
        break

    except tf.errors.NotFoundError:
      tf.logging.info("Checkpoint %s no longer exists, skipping.", ckpt)
Continuously loops while checking for checkpoints to evaluate.

  Args:
    estimator: Instance of Estimator.
    caption_input_fn: Estimator input function for supervised captioning.
    caption_eval_fn: Evaluation callback on captioning output.
    vqa_input_fn: Estimator input function for visual qa.
    vqa_eval_fn: Evaluation callback on visual qa output.
    max_checkpoint_number: Stop after this checkpoint is evaluated.
  
language/capwap/utils/experiment_utils.py