def marked_tokenize(self, text):
    """Return text tokenized with indicators if each token is reserved.

    Args:
      text: Text to tokenize

    Returns:
      List of text tokens and list of true/false, true if it is a reserved token
    """
    if self._reserved_re is None:
      tokens = self._tokenizer.tokenize(text)
      return tokens, len(tokens) * [False]
    else:
      reserved_tokens = list(re.finditer(self._compiled_reserved_re, text))
      if reserved_tokens:
        all_tokens = []
        all_marks = []
        position = 0
        for match in reserved_tokens:
          start = match.start()
          end = match.end()
          token = text[start:end]
          # This assumes a space based tokenizer, may not want to strip if that
          # changes
          pre_text = text[position:start].strip()
          if pre_text:
            tokenized = self._tokenizer.tokenize(pre_text)
            all_marks.extend(len(tokenized) * [False])
            all_tokens.extend(tokenized)
          all_marks.append(True)
          all_tokens.append(token)
          position = end
        if position != len(text):
          # This assumes a space based tokenizer, may not want to strip if that
          # changes
          post_text = text[position:].strip()
          if post_text:
            tokenized = self._tokenizer.tokenize(post_text)
            all_marks.extend(len(tokenized) * [False])
            all_tokens.extend(tokenized)
        return all_tokens, all_marks
      else:
        tokens = self._tokenizer.tokenize(text)
        return tokens, len(tokens) * [False]
Return text tokenized with indicators if each token is reserved.

    Args:
      text: Text to tokenize

    Returns:
      List of text tokens and list of true/false, true if it is a reserved token
    
language/serene/tokenizers.py