def convert_examples_to_features(examples, tokenizer, max_doc_length,
                                 doc_stride, max_query_length, output_fn):
  """Loads a data file into a list of `InputBatch`s."""

  unique_id = 1000000000

  for (example_index, example) in tqdm(enumerate(examples)):
    query_tokens = tokenizer.tokenize(example.question_text)

    if len(query_tokens) > max_query_length - 2:
      query_tokens = query_tokens[0:max_query_length - 2]  # -2 for [CLS], [SEP]

    tok_to_orig_index = []
    orig_to_tok_index = []
    all_doc_tokens = []
    for (i, token) in enumerate(example.doc_tokens):
      orig_to_tok_index.append(len(all_doc_tokens))
      sub_tokens = tokenizer.tokenize(token)
      for sub_token in sub_tokens:
        tok_to_orig_index.append(i)
        all_doc_tokens.append(sub_token)

    tok_start_position = None
    tok_end_position = None
    if example.is_impossible:
      tok_start_position = -1
      tok_end_position = -1
    if not example.is_impossible:
      tok_start_position = orig_to_tok_index[example.start_position]
      if example.end_position < len(example.doc_tokens) - 1:
        tok_end_position = orig_to_tok_index[example.end_position + 1] - 1
      else:
        tok_end_position = len(all_doc_tokens) - 1
      (tok_start_position, tok_end_position) = _improve_answer_span(
          all_doc_tokens, tok_start_position, tok_end_position, tokenizer,
          example.orig_answer_text)

    # The -2 accounts for [CLS] and [SEP]
    max_tokens_for_doc = max_doc_length - 2

    # We can have documents that are longer than the maximum sequence length.
    # To deal with this we do a sliding window approach, where we take chunks
    # of the up to our max length with a stride of `doc_stride`.
    _DocSpan = collections.namedtuple(  # pylint: disable=invalid-name
        "DocSpan", ["start", "length"])
    doc_spans = []
    start_offset = 0
    while start_offset < len(all_doc_tokens):
      length = len(all_doc_tokens) - start_offset
      if length > max_tokens_for_doc:
        length = max_tokens_for_doc
      doc_spans.append(_DocSpan(start=start_offset, length=length))
      if start_offset + length == len(all_doc_tokens):
        break
      start_offset += min(length, doc_stride)

    for (doc_span_index, doc_span) in enumerate(doc_spans):
      doc_tokens, qry_tokens = [], []
      doc_token_to_orig_map = {}
      doc_token_is_max_context = {}
      doc_segment_ids, qry_segment_ids = [], []

      # Question
      qry_tokens.append("[CLS]")
      qry_segment_ids.append(0)
      for token in query_tokens:
        qry_tokens.append(token)
        qry_segment_ids.append(0)
      qry_tokens.append("[SEP]")
      qry_segment_ids.append(0)

      # Document
      doc_tokens.append("[CLS]")
      doc_segment_ids.append(1)
      for i in range(doc_span.length):
        split_token_index = doc_span.start + i
        doc_token_to_orig_map[len(
            doc_tokens)] = tok_to_orig_index[split_token_index]

        is_max_context = _check_is_max_context(doc_spans, doc_span_index,
                                               split_token_index)
        doc_token_is_max_context[len(doc_tokens)] = is_max_context
        doc_tokens.append(all_doc_tokens[split_token_index])
        doc_segment_ids.append(1)
      doc_tokens.append("[SEP]")
      doc_segment_ids.append(1)

      doc_input_ids = tokenizer.convert_tokens_to_ids(doc_tokens)
      qry_input_ids = tokenizer.convert_tokens_to_ids(qry_tokens)

      # The mask has 1 for real tokens and 0 for padding tokens. Only real
      # tokens are attended to.
      doc_input_mask = [1] * len(doc_input_ids)
      qry_input_mask = [1] * len(qry_input_ids)

      # Zero-pad up to the sequence length.
      while len(doc_input_ids) < max_doc_length:
        doc_input_ids.append(0)
        doc_input_mask.append(0)
        doc_segment_ids.append(0)
      while len(qry_input_ids) < max_query_length:
        qry_input_ids.append(0)
        qry_input_mask.append(0)
        qry_segment_ids.append(0)

      assert len(doc_input_ids) == max_doc_length
      assert len(doc_input_mask) == max_doc_length
      assert len(doc_segment_ids) == max_doc_length
      assert len(qry_input_ids) == max_query_length
      assert len(qry_input_mask) == max_query_length
      assert len(qry_segment_ids) == max_query_length

      start_position = None
      end_position = None
      doc_start = doc_span.start
      doc_end = doc_span.start + doc_span.length - 1
      doc_offset = 1
      if not example.is_impossible:
        # For training, if our document chunk does not contain an annotation
        # we throw it out, since there is nothing to predict.
        out_of_span = False
        if not (tok_start_position >= doc_start and
                tok_end_position <= doc_end):
          out_of_span = True
        if out_of_span:
          start_position = 0
          end_position = 0
        else:
          start_position = tok_start_position - doc_start + doc_offset
          end_position = tok_end_position - doc_start + doc_offset
      if example.is_impossible:
        start_position = 0
        end_position = 0

      if example_index < 20:
        tf.logging.info("*** Example ***")
        tf.logging.info("unique_id: %s", unique_id)
        tf.logging.info("example_index: %s", example_index)
        tf.logging.info("doc_span_index: %s", doc_span_index)
        tf.logging.info(
            "doc_tokens: %s",
            " ".join([tokenization.printable_text(x) for x in doc_tokens]))
        tf.logging.info(
            "qry_tokens: %s",
            " ".join([tokenization.printable_text(x) for x in qry_tokens]))
        tf.logging.info(
            "doc_token_to_orig_map: %s", " ".join([
                "%d:%d" % (x, y)
                for (x, y) in six.iteritems(doc_token_to_orig_map)
            ]))
        tf.logging.info(
            "doc_token_is_max_context: %s", " ".join([
                "%d:%s" % (x, y)
                for (x, y) in six.iteritems(doc_token_is_max_context)
            ]))
        tf.logging.info("doc_input_ids: %s",
                        " ".join([str(x) for x in doc_input_ids]))
        tf.logging.info("doc_input_mask: %s",
                        " ".join([str(x) for x in doc_input_mask]))
        tf.logging.info("doc_segment_ids: %s",
                        " ".join([str(x) for x in doc_segment_ids]))
        tf.logging.info("qry_input_ids: %s",
                        " ".join([str(x) for x in qry_input_ids]))
        tf.logging.info("qry_input_mask: %s",
                        " ".join([str(x) for x in qry_input_mask]))
        tf.logging.info("qry_segment_ids: %s",
                        " ".join([str(x) for x in qry_segment_ids]))
        if example.is_impossible:
          tf.logging.info("impossible example")
        if not example.is_impossible:
          answer_text = " ".join(doc_tokens[start_position:(end_position + 1)])
          tf.logging.info("start_position: %d", start_position)
          tf.logging.info("end_position: %d", end_position)
          tf.logging.info("answer: %s",
                          tokenization.printable_text(answer_text))

      feature = InputFeatures(
          unique_id=unique_id,
          qas_id=example.qas_id.encode("utf-8"),
          example_index=example_index,
          doc_span_index=doc_span_index,
          doc_tokens=doc_tokens,
          doc_token_to_orig_map=doc_token_to_orig_map,
          doc_token_is_max_context=doc_token_is_max_context,
          doc_input_ids=doc_input_ids,
          doc_input_mask=doc_input_mask,
          doc_segment_ids=doc_segment_ids,
          qry_tokens=qry_tokens,
          qry_input_ids=qry_input_ids,
          qry_input_mask=qry_input_mask,
          qry_segment_ids=qry_segment_ids,
          start_position=start_position,
          end_position=end_position,
          is_impossible=example.is_impossible)

      # Run callback
      output_fn(feature)

      unique_id += 1
Loads a data file into a list of `InputBatch`s.
language/labs/drkit/hotpotqa/preprocessing/create_tfrecords.py