def convert_single_example(
    tydi_example: data.TyDiExample,
    tokenizer: tydi_tokenization_interface.TokenizerWithOffsets,
    is_training,
    max_question_length,
    max_seq_length,
    doc_stride,
    include_unknowns,
    errors,
    debug_info=None) -> List[InputFeatures]:
  """Converts a single `TyDiExample` into a list of InputFeatures.

  Args:
    tydi_example: `TyDiExample` from a single JSON line in the corpus.
    tokenizer: Tokenizer object that supports `tokenize` and
      `tokenize_with_offsets`.
    is_training: Are we generating these examples for training? (as opposed to
      inference).
    max_question_length: see FLAGS.max_question_length.
    max_seq_length: see FLAGS.max_seq_length.
    doc_stride: see FLAGS.doc_stride.
    include_unknowns: see FLAGS.include_unknowns.
    errors: List to be populated with error strings.
    debug_info: Dict to be populated with debugging information (e.g. how the
      strings were tokenized, etc.)

  Returns:
    List of `InputFeature`s.
  """
  features = []

  question_wordpieces = tokenizer.tokenize(tydi_example.question)
  # `tydi_example.contexts` includes the entire document (article) worth of
  # candidate passages concatenated with special tokens such as '[ContextId=0]'.
  all_doc_wp, contexts_start_offsets, contexts_end_offsets, offset_to_wp = (
      tokenizer.tokenize_with_offsets(tydi_example.contexts))

  # Check invariants.
  for i in contexts_start_offsets:
    if i > 0:
      assert i < len(tydi_example.context_to_plaintext_offset), (
          "Expected {} to be in `context_to_plaintext_offset` "
          "byte_len(contexts)={}".format(i,
                                         data.byte_len(tydi_example.contexts)))
  for i in contexts_end_offsets:
    if i > 0:
      assert i < len(tydi_example.context_to_plaintext_offset), (
          "Expected {} to be in `context_to_plaintext_offset` "
          "byte_len(contexts)={}".format(i,
                                         data.byte_len(tydi_example.contexts)))

  # The offsets `contexts_start_offsets` and `contexts_end_offsets` are
  # initially in terms of `tydi_example.contexts`, but we need them with regard
  # to the original plaintext from the input corpus.
  # `wp_start_offsets` and `wp_end_offsets` are byte-wise offsets with regard
  # to the original corpus plaintext.
  wp_start_offsets, wp_end_offsets = create_mapping(
      contexts_start_offsets, contexts_end_offsets,
      tydi_example.context_to_plaintext_offset)

  if len(question_wordpieces) > max_question_length:
    # Keeps only the last `max_question_length` wordpieces of the question.
    question_wordpieces = question_wordpieces[-max_question_length:]
  # Inserts the special question marker in front of the question.
  question_wordpieces.insert(0, tokenizer.get_vocab_id("[Q]"))
  if debug_info is not None:
    debug_info["query_wp_ids"] = question_wordpieces

  # DOCUMENT PROCESSING
  # The -3 accounts for
  # 1. [CLS] -- Special BERT class token, which is always first.
  # 2. [SEP] -- Special separator token, placed after question.
  # 3. [SEP] -- Special separator token, placed after article content.
  max_wordpieces_for_doc = max_seq_length - len(question_wordpieces) - 3
  assert max_wordpieces_for_doc >= 0
  if debug_info is not None:
    debug_info["all_doc_wp_ids"] = all_doc_wp

  # We can have documents that are longer than the maximum sequence length.
  # To deal with this we do a sliding window approach, where we take chunks
  # of up to our max length with a stride of `doc_stride`.
  doc_span = collections.namedtuple("DocSpan", ["start", "length"])
  doc_spans = []
  doc_span_start_wp_offset = 0

  while doc_span_start_wp_offset < len(all_doc_wp):
    length = len(all_doc_wp) - doc_span_start_wp_offset
    length = min(length, max_wordpieces_for_doc)
    doc_spans.append(doc_span(start=doc_span_start_wp_offset, length=length))
    if doc_span_start_wp_offset + length == len(all_doc_wp):
      break
    doc_span_start_wp_offset += min(length, doc_stride)

  # ANSWER PROCESSING
  if is_training:
    # Make sure these `TyDiExample`s were created with `is_training=True`.
    assert tydi_example.start_byte_offset is not None
    assert tydi_example.end_byte_offset is not None

    # First, check if we have any hope of finding a start/end wordpiece.
    # (If `offset_to_wp` is empty for this byte range, then it's pointless to
    # try to find a wordpiece range).
    # The `*_byte_offsets` in `tydi_example`.
    has_wordpiece = False
    for i in range(tydi_example.start_byte_offset,
                   tydi_example.end_byte_offset):
      if offset_to_wp.get(i, -1) >= 0:
        has_wordpiece = True
        break
    if not has_wordpiece:
      if debug_info is not None:
        searched_offset_to_wp = []
        for i in range(tydi_example.start_byte_offset,
                       tydi_example.end_byte_offset):
          searched_offset_to_wp.append(i)
        debug_info["offset_to_wp"] = offset_to_wp
        debug_info["searched_offset_to_wp"] = searched_offset_to_wp
      # It looks like the most likely cause of these issues is not having
      # whitespace between Latin/non-Latin scripts, which causes the tokenizer
      # to produce large chunks of non-wordpieced output. Unsurprisingly, the
      # vast majority of such problems arise in Thai and Japanese, which
      # typically do not use space between words.
      errors.append(
          "All byte indices between start/end offset have no assigned "
          "wordpiece.")
      return []

    # Find the indices of the first and last (inclusive) pieces of the answer
    # span. The end_byte_offset is exclusive, so we start our search for the
    # last piece from `end_byte_offset - 1` to ensure we do not select the piece
    # that follows the span.
    assert tydi_example.start_byte_offset <= tydi_example.end_byte_offset
    wp_start_position = find_nearest_wordpiece_index(
        tydi_example.start_byte_offset, offset_to_wp, scan_right=True)
    wp_end_position = find_nearest_wordpiece_index(
        tydi_example.end_byte_offset - 1, offset_to_wp, scan_right=False)

    # Sometimes there's no wordpiece at all for all the offsets in answer,
    # in such case, treat it as a null example.
    if wp_start_position == -1:
      errors.append("No starting wordpiece found.")
      return []
    if wp_end_position == -1:
      errors.append("No ending wordpiece found.")
      return []

  for doc_span_index, doc_span in enumerate(doc_spans):
    wps = []
    wps.append(tokenizer.get_vocab_id("[CLS]"))
    segment_ids = []
    segment_ids.append(0)
    wps.extend(question_wordpieces)
    segment_ids.extend([0] * len(question_wordpieces))
    wps.append(tokenizer.get_vocab_id("[SEP]"))
    segment_ids.append(0)

    wp_start_offset = [-1] * len(wps)
    wp_end_offset = [-1] * len(wps)

    for i in range(doc_span.length):
      split_token_index = doc_span.start + i
      wp_start_offset.append(wp_start_offsets[split_token_index])
      wp_end_offset.append(wp_end_offsets[split_token_index])
      wps.append(all_doc_wp[split_token_index])
      segment_ids.append(1)
    wps.append(tokenizer.get_vocab_id("[SEP]"))
    wp_start_offset.append(-1)
    wp_end_offset.append(-1)
    segment_ids.append(1)
    assert len(wps) == len(segment_ids)

    # The mask has 1 for real tokens and 0 for padding tokens. Only real
    # tokens are attended to.
    input_mask = [1] * len(wps)

    # Zero-pad up to the sequence length.
    padding_length = max_seq_length - len(wps)
    padding = [0] * padding_length
    padding_offset = [-1] * padding_length
    wps.extend(padding)
    input_mask.extend(padding)
    segment_ids.extend(padding)
    wp_start_offset.extend(padding_offset)
    wp_end_offset.extend(padding_offset)

    assert len(wps) == max_seq_length
    assert len(input_mask) == max_seq_length
    assert len(segment_ids) == max_seq_length
    assert len(wp_start_offset) == max_seq_length
    assert len(wp_end_offset) == max_seq_length

    start_position = None
    end_position = None
    answer_type = None
    answer_text = ""
    wp_error = False
    if is_training:
      doc_start = doc_span.start
      doc_end = doc_span.start + doc_span.length - 1
      # For training, if our document chunk does not contain an annotation
      # we throw it out, since there is nothing to predict.
      contains_an_annotation = (
          wp_start_position >= doc_start and wp_end_position <= doc_end)
      if ((not contains_an_annotation) or
          tydi_example.answer.type == data.AnswerType.UNKNOWN):
        # If an example has unknown answer type or does not contain the answer
        # span, then we only include it with probability --include_unknowns.
        # When we include an example with unknown answer type, we set the first
        # token of the passage to be the annotated short span.
        if (include_unknowns < 0 or random.random() > include_unknowns):
          continue
        start_position = 0
        end_position = 0
        answer_type = data.AnswerType.UNKNOWN
      else:
        doc_offset = len(question_wordpieces) + 2  #  one for CLS, one for SEP.

        start_position = wp_start_position - doc_start + doc_offset
        end_position = wp_end_position - doc_start + doc_offset
        answer_type = tydi_example.answer.type
        answer_start_byte_offset = wp_start_offset[start_position]
        answer_end_byte_offset = wp_end_offset[end_position]
        answer_text = tydi_example.contexts[
            answer_start_byte_offset:answer_end_byte_offset]

        try:
          assert answer_start_byte_offset > -1
          assert answer_end_byte_offset > -1
          assert end_position >= start_position
        except AssertionError:
          errors.append("wp_error")
          wp_error = True
          logging.info(wp_start_position, wp_end_position)
          logging.info(tydi_example.start_byte_offset,
                       tydi_example.end_byte_offset)
          logging.info(start_position, end_position)
          logging.info(doc_offset, doc_start)
          logging.info(tydi_example.example_id)
          logging.info("Error: end position smaller than start: %d",
                       tydi_example.example_id)

    feature = InputFeatures(
        unique_id=-1,  #  this gets assigned afterwards.
        example_index=tydi_example.example_id,
        language_id=tydi_example.language_id,
        doc_span_index=doc_span_index,
        wp_start_offset=wp_start_offset,
        wp_end_offset=wp_end_offset,
        input_ids=wps,
        input_mask=input_mask,
        segment_ids=segment_ids,
        start_position=start_position,
        end_position=end_position,
        answer_text=answer_text,
        answer_type=answer_type)
    if not wp_error:
      features.append(feature)

  return features
Converts a single `TyDiExample` into a list of InputFeatures.

  Args:
    tydi_example: `TyDiExample` from a single JSON line in the corpus.
    tokenizer: Tokenizer object that supports `tokenize` and
      `tokenize_with_offsets`.
    is_training: Are we generating these examples for training? (as opposed to
      inference).
    max_question_length: see FLAGS.max_question_length.
    max_seq_length: see FLAGS.max_seq_length.
    doc_stride: see FLAGS.doc_stride.
    include_unknowns: see FLAGS.include_unknowns.
    errors: List to be populated with error strings.
    debug_info: Dict to be populated with debugging information (e.g. how the
      strings were tokenized, etc.)

  Returns:
    List of `InputFeature`s.
  
language/canine/tydiqa/preproc.py