def tokenize(self, text):
    """Turn an english natural language string into tokens.

    Args:
      text: to transform

    Returns:
      tokens: list of string tokens
    """
    return py_utils.flatten_list(self._word_tokenizer.tokenize(s)
                                 for s in self._sent_tokenizer.tokenize(text))
Turn an english natural language string into tokens.

    Args:
      text: to transform

    Returns:
      tokens: list of string tokens
    
language/boolq/utils/tokenization.py