def convert_examples_to_features(
    tydi_examples, is_training,
    tokenizer: tydi_tokenization_interface.TokenizerWithOffsets,
    max_question_length, max_seq_length, doc_stride, include_unknowns,
    output_fn):
  """Converts `TyDiExample`s into `InputFeatures` and sends them to `output_fn`.

  Each entry is split into multiple `InputFeatures`, which contains windows

  spans of the article text that contain no more than N wordpieces so as to
  fit within BERT's context window.

  This function assigns `unique_ids` to features, which allow us to identify
  which example a shorter window of text corresponds to.

  Args:
    tydi_examples: generator of `TyDiExample`s generated by
      `read_tydi_examples`.
    is_training: boolean flag
    tokenizer: converts strings into input ids.
    max_question_length: see FLAGS.max_question_length.
    max_seq_length: see FLAGS.max_seq_length.
    doc_stride: see FLAGS.doc_stride.
    include_unknowns: see FLAGS.include_unknowns.
    output_fn: output function to be applied to the features generated from
      examples.

  Returns:
    num_spans_to_id: a dictionary containing a mapping from number of features
      to a list of example ids that has that number of features.
    num_examples: Number of examples from the `tydi_examples` generator.
  """
  num_spans_to_ids = collections.defaultdict(list)
  num_examples = 0
  for tydi_example in tydi_examples:
    example_index = tydi_example.example_id
    # Each TyDi entry is split into multiple features, each of
    # FLAGS.max_seq_length word pieces.
    errors = []
    features = convert_single_example(
        tydi_example,
        tokenizer=tokenizer,
        is_training=is_training,
        max_question_length=max_question_length,
        max_seq_length=max_seq_length,
        doc_stride=doc_stride,
        include_unknowns=include_unknowns,
        errors=errors)
    num_examples += 1

    num_spans_to_ids[len(features)].append(tydi_example.example_id)

    for feature in features:
      feature.example_index = example_index
      # This integer `unique_id` is used for compute_predictions
      # to merge features with example. Both `example_index` and
      # `doc_span_index` are integers, so this works primarily by virtue of
      # the `example_index`s being uniformly distributed with many unoccupied
      # indices between them so as to make collissions unlikely.
      feature.unique_id = feature.example_index + feature.doc_span_index
      output_fn(feature)

  return num_spans_to_ids, num_examples
Converts `TyDiExample`s into `InputFeatures` and sends them to `output_fn`.

  Each entry is split into multiple `InputFeatures`, which contains windows

  spans of the article text that contain no more than N wordpieces so as to
  fit within BERT's context window.

  This function assigns `unique_ids` to features, which allow us to identify
  which example a shorter window of text corresponds to.

  Args:
    tydi_examples: generator of `TyDiExample`s generated by
      `read_tydi_examples`.
    is_training: boolean flag
    tokenizer: converts strings into input ids.
    max_question_length: see FLAGS.max_question_length.
    max_seq_length: see FLAGS.max_seq_length.
    doc_stride: see FLAGS.doc_stride.
    include_unknowns: see FLAGS.include_unknowns.
    output_fn: output function to be applied to the features generated from
      examples.

  Returns:
    num_spans_to_id: a dictionary containing a mapping from number of features
      to a list of example ids that has that number of features.
    num_examples: Number of examples from the `tydi_examples` generator.
  
language/canine/tydiqa/preproc.py