def convert_examples_to_features(examples, tokenizer, max_seq_length,
                                 doc_stride, max_query_length, is_training,
                                 output_fn):
  """Loads a data file into a list of `InputBatch`s."""

  unique_id = 1000000000
  qtype_map = {"span": 0, "yes": 1, "no": 2}

  for (example_index, example) in enumerate(examples):
    if example_index % 1000 == 0:
      tf.logging.info("Processing %d", example_index)
    query_tokens = tokenizer.tokenize(example.question_text)

    if len(query_tokens) > max_query_length:
      query_tokens = query_tokens[0:max_query_length]

    qtype = qtype_map[example.question_type]

    all_doc_tokens = example.doc_tokens
    tok_to_orig_index = range(len(all_doc_tokens))

    tok_start_position = example.start_position
    tok_end_position = example.end_position
    tok_sup_positions = example.supporting
    tok_notsup_positions = example.notsupporting
    tok_pos_to_sp_fact = example.pos_to_sp_fact

    # The -3 accounts for [CLS], [SEP] and [SEP]
    max_tokens_for_doc = max_seq_length - len(query_tokens) - 3

    # We can have documents that are longer than the maximum sequence length.
    # To deal with this we do a sliding window approach, where we take chunks
    # of the up to our max length with a stride of `doc_stride`.
    _DocSpan = collections.namedtuple(  # pylint: disable=invalid-name
        "DocSpan", ["start", "length"])
    doc_spans = []
    start_offset = 0
    while start_offset < len(all_doc_tokens):
      length = len(all_doc_tokens) - start_offset
      if length > max_tokens_for_doc:
        length = max_tokens_for_doc
      doc_spans.append(_DocSpan(start=start_offset, length=length))
      if start_offset + length == len(all_doc_tokens):
        break
      start_offset += min(length, doc_stride)

    for (doc_span_index, doc_span) in enumerate(doc_spans):
      tokens = []
      token_to_orig_map = {}
      token_to_sp_fact_map = {}
      token_is_max_context = {}
      segment_ids = []
      tokens.append("[CLS]")
      segment_ids.append(0)
      for token in query_tokens:
        tokens.append(token)
        segment_ids.append(0)
      tokens.append("[SEP]")
      segment_ids.append(0)

      for i in range(doc_span.length):
        split_token_index = doc_span.start + i
        token_to_orig_map[len(tokens)] = tok_to_orig_index[split_token_index]

        is_max_context = _check_is_max_context(doc_spans, doc_span_index,
                                               split_token_index)
        token_is_max_context[len(tokens)] = is_max_context
        tokens.append(all_doc_tokens[split_token_index])
        segment_ids.append(1)
      tokens.append("[SEP]")
      segment_ids.append(1)

      input_ids = tokenizer.convert_tokens_to_ids(tokens)

      # The mask has 1 for real tokens and 0 for padding tokens. Only real
      # tokens are attended to.
      input_mask = [1] * len(input_ids)

      # Zero-pad up to the sequence length.
      while len(input_ids) < max_seq_length:
        input_ids.append(0)
        input_mask.append(0)
        segment_ids.append(0)

      assert len(input_ids) == max_seq_length
      assert len(input_mask) == max_seq_length
      assert len(segment_ids) == max_seq_length

      start_position = None
      end_position = None
      sup_positions = set()
      notsup_positions = set()
      if is_training and not example.is_impossible:
        # For training, if our document chunk does not contain an annotation
        # we throw it out, since there is nothing to predict.
        doc_offset = len(query_tokens) + 2
        doc_start = doc_span.start
        doc_end = doc_span.start + doc_span.length - 1
        out_of_span = False
        if not (tok_start_position >= doc_start and
                tok_end_position <= doc_end):
          out_of_span = True
        if out_of_span:
          start_position = 0
          end_position = 0
        else:
          start_position = tok_start_position - doc_start + doc_offset
          end_position = tok_end_position - doc_start + doc_offset
        # Check if any of the supporting facts are within range.
        for tok_sup_position in tok_sup_positions:
          if tok_sup_position >= doc_start and tok_sup_position <= doc_end:
            my_pos = tok_sup_position - doc_start + doc_offset
            sup_positions.add(my_pos)
            token_to_sp_fact_map[my_pos] = tok_pos_to_sp_fact[tok_sup_position]
        for tok_notsup_position in tok_notsup_positions:
          if (tok_notsup_position >= doc_start and
              tok_notsup_position <= doc_end):
            my_pos = tok_notsup_position - doc_start + doc_offset
            notsup_positions.add(my_pos)
            token_to_sp_fact_map[my_pos] = tok_pos_to_sp_fact[
                tok_notsup_position]

      if is_training and example.is_impossible:
        start_position = 0
        end_position = 0

      sup_labels, sup_mask = [], []
      for ii in range(max_seq_length):
        if ii in sup_positions:
          sup_labels.append(1)
          sup_mask.append(1)
        elif ii in notsup_positions:
          sup_labels.append(0)
          sup_mask.append(1)
        else:
          sup_labels.append(0)
          sup_mask.append(0)

      if example_index < 20:
        tf.logging.info("*** Example ***")
        tf.logging.info("unique_id: %s" % (unique_id))
        tf.logging.info("example_index: %s" % (example_index))
        tf.logging.info("doc_span_index: %s" % (doc_span_index))
        tf.logging.info("tokens: %s" % " ".join(
            [tokenization.printable_text(x) for x in tokens]))
        tf.logging.info("token_to_orig_map: %s" % " ".join(
            ["%d:%d" % (x, y) for (x, y) in six.iteritems(token_to_orig_map)]))
        tf.logging.info("token_is_max_context: %s" % " ".join([
            "%d:%s" % (x, y) for (x, y) in six.iteritems(token_is_max_context)
        ]))
        tf.logging.info("input_ids: %s" % " ".join([str(x) for x in input_ids]))
        tf.logging.info(
            "input_mask: %s" % " ".join([str(x) for x in input_mask]))
        tf.logging.info(
            "segment_ids: %s" % " ".join([str(x) for x in segment_ids]))
        if is_training and example.is_impossible:
          tf.logging.info("impossible example")
        if is_training and not example.is_impossible:
          answer_text = " ".join(tokens[start_position:(end_position + 1)])
          tf.logging.info("start_position: %d" % (start_position))
          tf.logging.info("end_position: %d" % (end_position))
          tf.logging.info("sup_labels: %r" % (sup_labels))
          tf.logging.info("sup_mask: %r" % (sup_mask))
          tf.logging.info("token_to_sp_fact_map: %r" % (token_to_sp_fact_map))
          tf.logging.info(
              "answer: %s" % (tokenization.printable_text(answer_text)))

      feature = InputFeatures(
          unique_id=unique_id,
          example_index=example_index,
          doc_span_index=doc_span_index,
          tokens=tokens,
          token_to_orig_map=token_to_orig_map,
          token_is_max_context=token_is_max_context,
          input_ids=input_ids,
          input_mask=input_mask,
          segment_ids=segment_ids,
          question_type=qtype,
          start_position=start_position,
          end_position=end_position,
          supporting_labels=sup_labels,
          supporting_mask=sup_mask,
          token_to_sp_fact_map=token_to_sp_fact_map,
          is_impossible=example.is_impossible)

      # Run callback
      output_fn(feature)

      unique_id += 1
Loads a data file into a list of `InputBatch`s.
language/labs/drkit/hotpotqa/answer_extractor.py