def create_tf_examples(tokenizer,
                       example,
                       image_string,
                       metadata,
                       is_test=False):
  """Creates TF examples for the given VCR example and image feature vector."""
  tokenize_fn = functools.partial(tokenize, tokenizer, example)
  q = tokenize_fn(example["question"])

  if FLAGS.include_rationales:
    if is_test:
      answers = [tokenize_fn(a) for a in example["answer_choices"]]
    else:
      # Only get true answer.
      a = example["answer_choices"][example["answer_label"]]
      answers = [tokenize_fn(a)]
    rationales = [tokenize_fn(r) for r in example["rationale_choices"]]
  else:
    answers = [tokenize_fn(a) for a in example["answer_choices"]]
    rationales = [[]]
  y = collections.OrderedDict()
  y["image"] = make_bytes_feature([image_string])

  annot_id = int(re.match(".*-([0-9]*)", example["annot_id"]).group(1))
  img_id = int(re.match(".*-([0-9]*)", example["img_id"]).group(1))
  y["annot_id"] = make_int64_feature([annot_id])
  y["img_id"] = make_int64_feature([img_id])

  for i, a in enumerate(answers):
    for j, r in enumerate(rationales):
      # Create text input.
      extra_tokens = []
      if FLAGS.append_all_bboxes:
        # Append bboxes to end of tokens.
        num_appended_bboxes = min(len(metadata["boxes"]), FLAGS.max_num_bboxes)
        for idx in range(num_appended_bboxes):
          extra_tokens.extend(tokenizer.tokenize(example["objects"][idx]))
          extra_tokens.append("[OBJ-%d]" % idx)

      max_len = FLAGS.max_seq_length - 4 - len(extra_tokens)
      q, a, r = cap_length(q, a, r, max_len)

      tokens = ["[CLS]", "[IMAGE]"] + q + ["[SEP]"] + a + r + ["[SEP]"]
      tokens.extend(extra_tokens)

      tf.logging.info("Final tokens: %s", " ".join(tokens))

      bbox_positions_matrix, bbox_indices_vector = get_bboxes(tokens, metadata)

      input_ids = tokenizer.convert_tokens_to_ids(tokens)
      segment_ids = [0] * (len(q) + 3) + [1] * (
          len(a) + len(r) + 1 + len(extra_tokens))
      input_mask = [1] * len(input_ids)

      # Pad.
      padding_len = FLAGS.max_seq_length - len(input_ids)
      input_ids.extend([0] * padding_len)
      segment_ids.extend([0] * padding_len)
      input_mask.extend([0] * padding_len)
      bbox_indices_vector.extend([-1] * padding_len)
      if (len(input_ids) != FLAGS.max_seq_length or
          len(segment_ids) != FLAGS.max_seq_length or
          len(input_mask) != FLAGS.max_seq_length or
          len(bbox_indices_vector) != FLAGS.max_seq_length):
        tf.logging.fatal("Bad feature lengths: %d, %d, %d, %d", len(input_ids),
                         len(segment_ids), len(input_mask),
                         len(bbox_indices_vector))

      # Is this the right choice?
      if "answer_label" in example and "rationale_label" in example:
        if FLAGS.include_rationales:
          positive = j == example["rationale_label"]
        else:
          positive = i == example["answer_label"]
      else:
        positive = False

      y["choice_id"] = make_int64_feature([i * 4 + j])
      y["input_ids"] = make_int64_feature(input_ids)
      y["segment_ids"] = make_int64_feature(segment_ids)
      y["input_mask"] = make_int64_feature(input_mask)
      y["label"] = make_int64_feature([int(positive)])
      y["bbox_pos"] = make_int64_feature(bbox_positions_matrix)
      y["bbox_idx"] = make_int64_feature(bbox_indices_vector)
      yield tf.train.Example(features=tf.train.Features(feature=y))
Creates TF examples for the given VCR example and image feature vector.
language/question_answering/b2t2/compute_vcr_features.py