def read_tydi_examples(
    input_file, tokenizer: tydi_tokenization_interface.TokenizerWithOffsets,
    is_training, max_passages, max_position, fail_on_invalid, open_fn):
  """Read a TyDi json file into a list of `TyDiExample`.

  Delegates to `preproc.create_entry_from_json` to add special tokens to
  input and handle character offset tracking.

  Args:
    input_file: Path or glob to input JSONL files to be read (possibly gzipped).
    tokenizer: Used to create special marker symbols to insert into the text.
    is_training: Should we create training samples? (as opposed to eval
      samples).
    max_passages: See FLAGS.max_passages.
    max_position: See FLAGS.max_position.
    fail_on_invalid: Should we immediately stop processing if an error is
      encountered?
    open_fn: A function that returns a file object given a path. Usually
      `tf_io.gopen`; could be standard Python `open` if using this module
      outside Tensorflow.

  Yields:
    `TyDiExample`s
  """
  input_paths = glob.glob(input_file)
  if not input_paths:
    raise ValueError("No paths matching glob '{}'".format(input_file))

  non_valid_count = 0
  n = 0
  for path in input_paths:
    logging.info("Reading: %s", path)
    with open_fn(path) as input_file:
      logging.info(path)
      for line in input_file:
        json_dict = json.loads(line, object_pairs_hook=collections.OrderedDict)
        entry = create_entry_from_json(
            json_dict,
            tokenizer,
            max_passages=max_passages,
            max_position=max_position,
            fail_on_invalid=fail_on_invalid)
        if entry:
          tydi_example = data.to_tydi_example(entry, is_training)
          n += 1
          yield tydi_example
        else:
          if fail_on_invalid:
            raise ValueError("Found invalid example.")
          non_valid_count += 1

  if n == 0:
    raise ValueError(
        "No surviving examples from input_file '{}'".format(input_file))

  logging.info("*** # surviving examples %d ***", n)
  logging.info("*** # pruned examples %d ***", non_valid_count)
Read a TyDi json file into a list of `TyDiExample`.

  Delegates to `preproc.create_entry_from_json` to add special tokens to
  input and handle character offset tracking.

  Args:
    input_file: Path or glob to input JSONL files to be read (possibly gzipped).
    tokenizer: Used to create special marker symbols to insert into the text.
    is_training: Should we create training samples? (as opposed to eval
      samples).
    max_passages: See FLAGS.max_passages.
    max_position: See FLAGS.max_position.
    fail_on_invalid: Should we immediately stop processing if an error is
      encountered?
    open_fn: A function that returns a file object given a path. Usually
      `tf_io.gopen`; could be standard Python `open` if using this module
      outside Tensorflow.

  Yields:
    `TyDiExample`s
  
language/canine/tydiqa/preproc.py