def _compute_token_boundaries(self, text, token_strs):
    """Computes the byte offset boundaries for each token in a text.

    Uses greedy left-to-right string-matching to align each token with a
    position in the text.

    Args:
      text (bytes): a byte string.
      token_strs (list[str]): a list of token strings produced by the BERT
        tokenizer.

    Returns:
      A list of (start, stop) byte offsets (right-exclusive), one for each
      token.

    Raises:
      TokenizationError: if we fail to identify the byte offset for a token.
    """
    text = tokenization.convert_to_unicode(text)
    c2b = self._get_char_to_byte_map(text)

    # Normalization to match tokens.
    normalized_text = (
        self._base_tokenizer.basic_tokenizer._run_strip_accents(text))  # pylint: disable=protected-access
    if self.do_lower_case:
      normalized_text = normalized_text.lower()

    # NOTE: this is a crude check to see if lowercasing and accent-stripping
    # have altered character byte offsets. It is not perfect.
    if len(normalized_text) != len(text):
      raise TokenizationError('Normalization affected character byte offsets.')

    # Strip wordpiece prefixes.
    token_strs = [t[2:] if t.startswith('##') else t for t in token_strs]

    # Align each token with its position in normalized_text
    search_start = 0
    token_char_boundaries = []
    for token_str in token_strs:
      token_start = normalized_text.find(token_str, search_start)
      token_stop = token_start + len(token_str)
      if token_start == -1:
        raise TokenizationError('Cannot align "{}" in text: {}'.format(
            token_str, normalized_text))

      token_char_boundaries.append((token_start, token_stop))
      search_start = token_stop  # Start searching right after this token.

    # Convert char boundaries to byte boundaries
    token_byte_boundaries = [
        (c2b[start], c2b[stop]) for start, stop in token_char_boundaries
    ]
    return token_byte_boundaries
Computes the byte offset boundaries for each token in a text.

    Uses greedy left-to-right string-matching to align each token with a
    position in the text.

    Args:
      text (bytes): a byte string.
      token_strs (list[str]): a list of token strings produced by the BERT
        tokenizer.

    Returns:
      A list of (start, stop) byte offsets (right-exclusive), one for each
      token.

    Raises:
      TokenizationError: if we fail to identify the byte offset for a token.
    
language/realm/featurization.py