def get_vocabularies(data_dir):
  """Get input and target vocabularies.

  If will read vocabularies from file if exists. Otherwise,
  it will generating vocabulary from examples.

  Args:
    data_dir: Path to data dir.

  Returns:
    Input and target vocabularies.
  """
  input_vocab_file = os.path.join(data_dir, 'training_input_vocab.txt')
  target_vocab_file = os.path.join(data_dir, 'training_target_vocab.txt')
  if tf.io.gfile.exists(input_vocab_file) and tf.io.gfile.exists(
      target_vocab_file):
    logging.info('Loading input vocabulary from %s.', data_dir)
    input_vocab = Vocabulary.load(input_vocab_file)
    target_vocab = Vocabulary.load(target_vocab_file)
  else:
    logging.info('Generating input vocabulary.')
    input_vocab = Vocabulary()
    target_vocab = Vocabulary()
    examples = tfds.load(f'grounded_scan/{FLAGS.split}', split='train')
    for example in examples:
      commands = _parse_commands(example['command'])
      target_commands = _parse_commands(example['target_commands'])
      if FLAGS.split == 'spatial_relation_splits':
        # The spatial relation split also uses whitespace as separtor,
        # therefore it will tokenize south east as two words.
        commands = re.split(',| ', ','.join(commands))
      input_vocab.add_sentence(commands)
      target_vocab.add_sentence(target_commands)
    input_vocab.save(input_vocab_file)
    target_vocab.save(target_vocab_file)
    logging.info('Saved input vocabulary to %s', data_dir)
  return input_vocab, target_vocab
Get input and target vocabularies.

  If will read vocabularies from file if exists. Otherwise,
  it will generating vocabulary from examples.

  Args:
    data_dir: Path to data dir.

  Returns:
    Input and target vocabularies.
  
language/gscan/xattn_model/dataset/preprocess.py