def to_tokens(text, tokenizer):
  """Tokenize the text and return mapping from chars to tokens."""
  def is_whitespace(c):
    if c == " " or c == "\t" or c == "\r" or c == "\n" or ord(c) == 0x202F:
      return True
    return False
  doc_tokens = []
  char_to_word_offset = []
  prev_is_whitespace = True
  for c in text:
    if is_whitespace(c):
      prev_is_whitespace = True
    else:
      if prev_is_whitespace:
        doc_tokens.append(c)
      else:
        doc_tokens[-1] += c
      prev_is_whitespace = False
    char_to_word_offset.append(len(doc_tokens) - 1)
  orig_to_tok_index = []
  all_doc_tokens = []
  for token in doc_tokens:
    orig_to_tok_index.append(len(all_doc_tokens))
    sub_tokens = tokenizer.tokenize(token)
    for sub_token in sub_tokens:
      all_doc_tokens.append(sub_token)
  char_to_final_offset = []
  for c in char_to_word_offset:
    char_to_final_offset.append(orig_to_tok_index[c])
  return all_doc_tokens, char_to_final_offset
Tokenize the text and return mapping from chars to tokens.
language/labs/drkit/hotpotqa/demo.py