def normalize_text(input_text):
  """Lower text and remove punctuation, articles and extra whitespace."""

  def remove_articles(s):
    return re.sub(r'\b(a|an|the)\b', ' ', s)

  def replace_punctuation(s):
    to_replace = set(string.punctuation)
    # TODO(danielandor): Fix annotation tokenization artifacts here as well.
    return ''.join('' if ch in to_replace else ch for ch in s)

  def white_space_fix(s):
    return ' '.join(s.split())

  text = input_text.lower()
  text = replace_punctuation(text)
  text = remove_articles(text)
  text = white_space_fix(text)
  return text
Lower text and remove punctuation, articles and extra whitespace.
language/decontext/eval_util.py