def get_lang_counts(input_jsonl_pattern):
  """Gets the number of examples for each language."""
  lang_dict = collections.Counter()
  for input_path in tf.gfile.Glob(input_jsonl_pattern):
    with gzip.GzipFile(fileobj=tf.gfile.Open(input_path)) as input_file:  # pytype: disable=wrong-arg-types
      for line in input_file:
        json_elem = json.loads(line)
        lang_dict[json_elem["language"]] += 1
  return lang_dict
Gets the number of examples for each language.
language/canine/tydiqa/prepare_tydi_data.py