def spider_template_fn(target):
  """Anonymize quoted substrings and numbers in SQL."""
  # First, replace any numeric token.
  tokens = sql_tokenizer.tokenize_sql(target)
  template_tokens = []
  for token in tokens:
    if is_number(token):
      template_tokens.append(PLACEHOLDER)
    else:
      template_tokens.append(token)
  template = " ".join(template_tokens)

  # Second, replace any subspan surrounded by single or double quotes.
  in_quotes = False
  quote_token = None
  new_template = ""
  for char in template:
    if in_quotes:
      if char == quote_token:
        in_quotes = False
        quote_token = None
    else:
      if char in ("'", "\""):
        in_quotes = True
        quote_token = char
        new_template += PLACEHOLDER
      else:
        new_template += char
  return new_template
Anonymize quoted substrings and numbers in SQL.
language/compgen/nqg/tasks/spider/gen_template_split.py