def write_tf_feature_files(self, tydi_examples_iter):
    """Converts TyDi examples to features and writes them to files."""
    logging.info("Converting examples started.")

    total_feature_count_frequencies = collections.defaultdict(int)
    total_num_examples = 0
    total_num_features = 0
    if self.predict_file_shard_size:
      shard_iter = self.sharded_iterator(tydi_examples_iter,
                                         self.predict_file_shard_size)
    else:
      # No sharding, so treat the whole input as one "shard".
      shard_iter = [tydi_examples_iter]
    for shard_num, examples in enumerate(shard_iter, 1):
      features_writer = tf_io.FeatureWriter(
          filename=os.path.join(self.output_dir,
                                "features.tf_record-%03d" % shard_num),
          is_training=False)
      num_features_to_ids, shard_num_examples = (
          preproc.convert_examples_to_features(
              tydi_examples=examples,
              tokenizer=self.get_tokenizer(),
              is_training=False,
              max_question_length=self.max_question_length,
              max_seq_length=self.max_seq_length,
              doc_stride=self.doc_stride,
              include_unknowns=self.include_unknowns,
              output_fn=features_writer.process_feature))
      features_writer.close()

      if shard_num_examples == 0:
        continue

      shard_num_features = 0
      for num_features, ids in num_features_to_ids.items():
        shard_num_features += (num_features * len(ids))
        total_feature_count_frequencies[num_features] += len(ids)
      total_num_examples += shard_num_examples
      total_num_features += shard_num_features
      logging.info("Shard %d: Converted %d input examples into %d features.",
                   shard_num, shard_num_examples, shard_num_features)
      logging.info("  Total so far: %d input examples, %d features.",
                   total_num_examples, total_num_features)
      yield (shard_num, (features_writer.filename, shard_num_examples,
                         shard_num_features))

    logging.info("Converting examples finished.")
    logging.info("  Total examples = %d", total_num_examples)
    logging.info("  Total features = %d", total_num_features)
    logging.info("  total_feature_count_frequencies = %s",
                 sorted(total_feature_count_frequencies.items()))
Converts TyDi examples to features and writes them to files.
language/canine/tydiqa/run_tydi_lib.py