def process_target(target):
  """Preprocess target for space-separated tokenization."""
  target_sql_tokens = sql_tokenizer.tokenize_sql(target)
  target = " ".join(target_sql_tokens)
  target = _split_punc(target)
  # Split punc twice, to handle "%foo%" wrapped in two punc chars.
  # TODO(petershaw): Update _split_punc to correctly handle this case with
  # a single invocation.
  target = _split_punc(target)
  # Remove extra whitespace.
  target = " ".join(target.split())
  return target
Preprocess target for space-separated tokenization.
language/compgen/nqg/tasks/spider/nqg_tokenization.py