def get_dataset(params, mode, caption_file, qa_file, scratch_file, vocab):
  """Gets a tf.data.Dataset with RC data for eval. Processes data online."""
  assert mode == tf_estimator.ModeKeys.PREDICT

  # Create writer for temp tfrecord data.
  tf.logging.info("Converting examples to TFRecords...")
  writer = tf.io.TFRecordWriter(scratch_file)
  count = 0
  for example in iter_rc_examples(caption_file, qa_file, vocab):
    count += 1
    writer.write(example.SerializeToString())
    if count % 10000 == 0:
      tf.logging.info("Serialized %d examples...", count)
  writer.close()
  tf.logging.info("Serialized %d examples.", count)

  dataset = tf.data.TFRecordDataset(scratch_file, buffer_size=16 * 1024 * 1024)
  dataset = dataset.map(
      parse_example, num_parallel_calls=params["num_input_threads"])
  dataset = dataset.map(
      functools.partial(
          preprocess_mapper,
          params=params,
          lookup_table=vocab.get_string_lookup_table()),
      num_parallel_calls=params["num_input_threads"])

  # Batch.
  dataset = dataset.batch(params["batch_size"], drop_remainder=False)
  dataset = dataset.prefetch(params["prefetch_batches"])

  return dataset
Gets a tf.data.Dataset with RC data for eval. Processes data online.
language/capwap/datasets/rc_dataset.py