def convert_single_example(ex_index, example, max_seq_length, tokenizer):
  """Converts a single `InputExample` into a single `InputFeatures`."""
  # Add padding examples here

  example_type = collections.namedtuple(
      "Example", ["input_ids", "input_mask", "segment_ids", "labels"])

  sents = [example["ctx"]] + example["endings"]
  sents = [tokenizer.tokenize(sent) for sent in sents]

  for ending_i in range(1, 5):
    assert len(sents[0]) + len(sents[ending_i]) < (max_seq_length -
                                                   3), "Example is too long"

  input_id_list, sengment_id_list, input_mask_list = [], [], []
  for i in range(4):
    tokens = []
    segment_ids = []
    tokens.append("[CLS]")
    segment_ids.append(0)
    for token in sents[0]:
      tokens.append(token)
      segment_ids.append(0)
    tokens.append("[SEP]")
    segment_ids.append(0)

    for token in sents[i + 1]:
      tokens.append(token)
      segment_ids.append(1)
    tokens.append("[SEP]")
    segment_ids.append(0)

    input_ids = tokenizer.convert_tokens_to_ids(tokens)

    # The mask has 1 for real tokens and 0 for padding tokens. Only real
    # tokens are attended to.
    input_mask = [1] * len(input_ids)

    # Zero-pad up to the sequence length.
    while len(input_ids) < max_seq_length:
      input_ids.append(0)
      input_mask.append(0)
      segment_ids.append(0)

    assert len(input_ids) == max_seq_length
    assert len(input_mask) == max_seq_length
    assert len(segment_ids) == max_seq_length

    input_id_list.append(input_ids)
    sengment_id_list.append(segment_ids)
    input_mask_list.append(input_mask)

  labels = [0] * 8
  labels[4] = example["label"]

  if ex_index < 5:
    tf.logging.info("*** Example ***")
    tf.logging.info("tokens: %s" % " ".join(tokens))
    tf.logging.info("input_ids: %s" % " ".join([str(x) for x in input_ids]))
    tf.logging.info("input_mask: %s" % " ".join([str(x) for x in input_mask]))
    tf.logging.info("segment_ids: %s" % " ".join([str(x) for x in segment_ids]))

  feature = example_type(input_id_list, input_mask_list, sengment_id_list,
                         labels)
  return feature
Converts a single `InputExample` into a single `InputFeatures`.
language/conpono/evals/model_builder.py