def parsing_token_generator(data_dir, tmp_dir, train, source_vocab_size,
                            target_vocab_size):
  """Generator for parsing as a sequence-to-sequence task that uses tokens.

  This generator assumes the files parsing_{train,dev}.trees, which contain
  trees in WSJ format.

  Args:
    data_dir: path to the data directory.
    tmp_dir: path to temporary storage directory.
    train: whether we're training or not.
    source_vocab_size: source vocab size.
    target_vocab_size: target vocab size.

  Returns:
    A generator to a dictionary of inputs and outputs.
  """
  # TODO(lukaszkaiser): Correct these calls to generate vocabularies. No data
  # sources are being passed.
  del (data_dir, tmp_dir, train, source_vocab_size, target_vocab_size)
  assert False, "Vocabulary generation not implemented"
  # source_symbolizer_vocab = generator_utils.get_or_generate_vocab(
  #     data_dir, tmp_dir, "wsj_source.vocab.%d" % source_vocab_size,
  #     source_vocab_size)
  # target_symbolizer_vocab = generator_utils.get_or_generate_vocab(
  #     data_dir, tmp_dir, "wsj_target.vocab.%d" % target_vocab_size,
  #     target_vocab_size)
  # filename = "%s_%s.trees" % (FLAGS.parsing_path, "train" if train else "dev")
  # tree_filepath = os.path.join(tmp_dir, filename)
  # return token_generator(tree_filepath, source_symbolizer_vocab,
  #                        target_symbolizer_vocab, 1)
Generator for parsing as a sequence-to-sequence task that uses tokens.

  This generator assumes the files parsing_{train,dev}.trees, which contain
  trees in WSJ format.

  Args:
    data_dir: path to the data directory.
    tmp_dir: path to temporary storage directory.
    train: whether we're training or not.
    source_vocab_size: source vocab size.
    target_vocab_size: target vocab size.

  Returns:
    A generator to a dictionary of inputs and outputs.
  
tensor2tensor/data_generators/wsj_parsing.py