def _convert_to_tf_example(example, tokenizer, rules, config, max_sizes=None):
  """Return tf.Example generated for input (source, target)."""
  source = example[0]
  target = example[1]
  tokens = source.split(" ")
  num_tokens = len(tokens)

  # Tokenize.
  (wordpiece_ids, num_wordpieces, token_start_wp_idx,
   token_end_wp_idx) = tokenization_utils.get_wordpiece_inputs(
       tokens, tokenizer)

  # Run chart parser.
  target_node = parsing_utils.get_target_node(source, target, rules)
  if not target_node:
    raise ValueError("No parse returned for target for example: (%s, %s)" %
                     (source, target))
  merged_node = parsing_utils.get_merged_node(source, rules)

  # Get anchored applications.
  rule_to_idx_map = get_rule_to_idx_map(rules)
  (application_key_to_idx_map, application_span_begin, application_span_end,
   application_rule_idx) = _get_applications(merged_node, rule_to_idx_map,
                                             token_start_wp_idx,
                                             token_end_wp_idx)
  num_applications = len(application_span_begin)

  def application_idx_fn(span_begin, span_end, rule):
    return application_key_to_idx_map[(span_begin, span_end, rule)]

  # Get numerator forest.
  (nu_node_type, nu_node_1_idx, nu_node_2_idx, nu_application_idx,
   nu_num_nodes) = forest_serialization.get_forest_lists(
       target_node, num_tokens, application_idx_fn)
  # Get denominator forest.
  (de_node_type, de_node_1_idx, de_node_2_idx, de_application_idx,
   de_num_nodes) = forest_serialization.get_forest_lists(
       merged_node, num_tokens, application_idx_fn)

  # Create features dict.
  features = collections.OrderedDict()
  features["wordpiece_ids"] = _create_int_feature(wordpiece_ids,
                                                  config["max_num_wordpieces"])
  features["num_wordpieces"] = _create_int_feature([num_wordpieces], 1)

  features["application_span_begin"] = _create_int_feature(
      application_span_begin, config["max_num_applications"])
  features["application_span_end"] = _create_int_feature(
      application_span_end, config["max_num_applications"])
  features["application_rule_idx"] = _create_int_feature(
      application_rule_idx, config["max_num_applications"])

  features["nu_node_type"] = _create_int_feature(
      nu_node_type, config["max_num_numerator_nodes"])
  features["nu_node_1_idx"] = _create_int_feature(
      nu_node_1_idx, config["max_num_numerator_nodes"])
  features["nu_node_2_idx"] = _create_int_feature(
      nu_node_2_idx, config["max_num_numerator_nodes"])
  features["nu_application_idx"] = _create_int_feature(
      nu_application_idx, config["max_num_numerator_nodes"])
  features["nu_num_nodes"] = _create_int_feature([nu_num_nodes], 1)

  features["de_node_type"] = _create_int_feature(
      de_node_type, config["max_num_denominator_nodes"])
  features["de_node_1_idx"] = _create_int_feature(
      de_node_1_idx, config["max_num_denominator_nodes"])
  features["de_node_2_idx"] = _create_int_feature(
      de_node_2_idx, config["max_num_denominator_nodes"])
  features["de_application_idx"] = _create_int_feature(
      de_application_idx, config["max_num_denominator_nodes"])
  features["de_num_nodes"] = _create_int_feature([de_num_nodes], 1)

  tf_example = tf.train.Example(features=tf.train.Features(feature=features))

  # Update max sizes.
  if max_sizes is not None:
    max_sizes["num_wordpieces"] = max(max_sizes["num_wordpieces"],
                                      num_wordpieces)
    max_sizes["num_applications"] = max(max_sizes["num_applications"],
                                        num_applications)
    max_sizes["nu_num_nodes"] = max(max_sizes["nu_num_nodes"], nu_num_nodes)
    max_sizes["de_num_nodes"] = max(max_sizes["de_num_nodes"], de_num_nodes)

  return tf_example
Return tf.Example generated for input (source, target).
language/compgen/nqg/model/parser/data/example_converter.py