def preprocess_text(text, tokenizer):
  """Tokenize and convert raw text to ids.

  Args:
    text: String to be tokenized.
    tokenizer: An instance of Tokenizer class.

  Returns:
    tokens: List of strings.
    char_to_token_map: List of token ids mapping each character in the original
      string to the first and last index of the split tokens it appears in.
  """
  # Split by whitespace.
  wspace_tokens = []
  char_to_wspace = []
  prev_is_whitespace = True
  for c in text:
    if is_whitespace(c):
      prev_is_whitespace = True
    else:
      if prev_is_whitespace:
        wspace_tokens.append(c)
      else:
        wspace_tokens[-1] += c
      prev_is_whitespace = False
    char_to_wspace.append(len(wspace_tokens) - 1)

  # Tokenize each split token.
  orig_to_tok_index = []
  tokens = []
  for token in wspace_tokens:
    orig_to_tok_index.append(len(tokens))
    sub_tokens = tokenizer.tokenize(token)
    for sub_token in sub_tokens:
      tokens.append(sub_token)

  # Map characters to tokens.
  char_to_token_map = []
  for cc in char_to_wspace:
    st = orig_to_tok_index[cc]
    if cc == len(orig_to_tok_index) - 1:
      en = len(tokens) - 1
    else:
      en = orig_to_tok_index[cc + 1] - 1
    char_to_token_map.append((st, en))

  return tokens, char_to_token_map
Tokenize and convert raw text to ids.

  Args:
    text: String to be tokenized.
    tokenizer: An instance of Tokenizer class.

  Returns:
    tokens: List of strings.
    char_to_token_map: List of token ids mapping each character in the original
      string to the first and last index of the split tokens it appears in.
  
language/labs/drkit/bert_utils.py