def continuous_eval(eval_tfrecord_filename, estimator, eval_examples,
                    eval_features):
  """Run continuous evaluation on given TFRecords file."""
  current_ckpt = None
  stop_evaluating = False
  num_waits = 0
  best_f = 0.
  f_log = tf.gfile.Open(os.path.join(FLAGS.output_dir, "scores.log"), "w")
  if not tf.gfile.Exists(os.path.join(FLAGS.output_dir, "eval")):
    tf.gfile.MakeDirs(os.path.join(FLAGS.output_dir, "eval"))
  event_writer = tf.summary.FileWriter(os.path.join(FLAGS.output_dir, "eval"))
  while not stop_evaluating:
    if FLAGS.use_best_ckpt_for_predict:
      ckpt_path = os.path.join(FLAGS.output_dir, "best_model")
      if not os.path.exists(ckpt_path + ".meta"):
        tf.logging.info("No best_model checkpoint found in %s",
                        FLAGS.output_dir)
        tf.logging.info("Skipping evaluation.")
        break
      output_prediction_file = os.path.join(FLAGS.output_dir,
                                            "predictions.json")
      output_nbest_file = os.path.join(FLAGS.output_dir,
                                       "nbest_predictions.json")
      output_null_log_odds_file = os.path.join(FLAGS.output_dir,
                                               "null_odds.json")
      stop_evaluating = True
    else:
      ckpt_path = tf.train.latest_checkpoint(FLAGS.output_dir)
      if ckpt_path is None:
        tf.logging.info("No checkpoint in %s", FLAGS.output_dir)
        tf.logging.info("Waiting for 100s")
        time.sleep(100)
        continue
      if ckpt_path == current_ckpt:
        tf.logging.info("No new checkpoint in %s", FLAGS.output_dir)
        num_waits += 1
        if num_waits == 15:
          tf.logging.info("Waited for 1000s, exiting now.")
          # stop_evaluating = True
        else:
          tf.logging.info("Waiting for 100s")
          time.sleep(100)
        continue
      num_waits = 0
      current_ckpt = ckpt_path
      model_name = os.path.basename(ckpt_path)
      output_prediction_file = os.path.join(FLAGS.output_dir,
                                            "predictions_%s.json" % model_name)
      output_nbest_file = os.path.join(FLAGS.output_dir,
                                       "nbest_predictions_%s.json" % model_name)
      output_null_log_odds_file = os.path.join(FLAGS.output_dir,
                                               "null_odds_%s.json" % model_name)
      output_relationwise_file = os.path.join(
          FLAGS.output_dir, "relationwise_%s.json" % model_name)

    single_eval(eval_tfrecord_filename, estimator, eval_examples, eval_features,
                ckpt_path, output_prediction_file, output_nbest_file,
                output_null_log_odds_file)

    micro, macro, relationwise, _ = evaluate.compute_scores(
        FLAGS.predict_file, output_prediction_file)
    message = (
        "Model %s Micro-P %.3f Micro-R %.3f Micro-F %.3f "
        "Macro-P %.3f Macro-R %.3f Macro-F %.3f" %
        (ckpt_path, micro[0], micro[1], micro[2], macro[0], macro[1], macro[2]))
    tf.logging.info(message)
    f_log.write(message + "\n")
    f_log.flush()
    ckpt_number = int(ckpt_path.rsplit("-", 1)[1])
    micro_f_summary = tf.Summary(value=[
        tf.Summary.Value(tag="micro_f1", simple_value=micro[2]),
    ])
    event_writer.add_summary(micro_f_summary, global_step=ckpt_number)
    macro_f_summary = tf.Summary(value=[
        tf.Summary.Value(tag="macro_f1", simple_value=macro[2]),
    ])
    event_writer.add_summary(macro_f_summary, global_step=ckpt_number)
    for rel, (metrics, _) in relationwise.items():
      rel_summary = tf.Summary(value=[
          tf.Summary.Value(tag="relation/" + rel, simple_value=metrics[2]),
      ])
      event_writer.add_summary(rel_summary, global_step=ckpt_number)
    json.dump(
        {key: val[0].tolist() + [val[1]] for key, val in relationwise.items()},
        tf.gfile.Open(output_relationwise_file, "w"))
    if micro[2] > best_f and ckpt_path is not None:
      best_f = micro[2]
      _copy_model(ckpt_path, os.path.join(FLAGS.output_dir, "best_model"))
  f_log.close()
Run continuous evaluation on given TFRecords file.
language/labs/drkit/run_dualencoder_lsf.py