def tokenize_with_mention_spans(
    tokenizer: Tokenizer,
    sentence: str,
    spans: List[Tuple[int, int]],
    max_length: int,
    add_bert_tokens: bool = True,
    allow_truncated_spans: bool = False
) -> Tuple[List[str], List[int], List[int], List[Tuple[int, int]], List[int]]:
  """Tokenizes and resolves byte-offsets to word-offsets.

  Given a sentence and a set of inclusive byte-offset spans, applies a BERT
  tokenizer to the sentence and returns inclusive word-piece spans.

  Args:
    tokenizer: a BERT tokenizer
    sentence: a string, sentence to be tokenized
    spans: [first token, last token] byte-offset spans. The spans should be
      sorted by start_offset, and should also be non-overlapping.
    max_length: the maximum length to truncate or pad to.
    add_bert_tokens: whether to add special BERT tokens (CLS, SEP) to the input.
    allow_truncated_spans: when it is okay to drop spans due to length. This
      should be false for official evals!

  Returns:
    word_pieces: tokenized word pieces.
    word_piece_ids: IDs of tokenized word pieces.
    word_piece_mask: mask of word pieces (for padding).
    token_spans: inclusive word-piece spans of the input byte-offset spans.
    span_indexes: indexes with respect to the input spans corresponding to
      output token spans. This is necessary because some input spans can be
      truncated or otherwise dropped.

  Raises:
    RuntimeError: if allow_truncated_spans is False but spans are dropped due
      to length.
  """
  sentence = sentence.encode("utf-8")

  last_index = 0
  last_start = 0
  word_pieces = []
  if add_bert_tokens:
    word_pieces.append("[CLS]")

  token_spans = []
  span_indexes = []
  for i, span in enumerate(spans):
    if span[0] < last_start:
      raise ValueError("Spans not in sequential order: %s" % (str(spans)))
    if span[1] < span[0]:
      raise ValueError("Span end before start: %s" % (str(span)))
    if span[0] < last_index:
      continue
    pre_span_string = sentence[last_index:span[0]]
    span_string = sentence[span[0]:span[1] + 1]
    last_index = span[1] + 1
    last_start = span[0]

    pre_span_word_pieces = tokenizer.tokenize(pre_span_string)
    word_pieces += pre_span_word_pieces
    span_token_start = len(word_pieces)
    span_word_pieces = tokenizer.tokenize(span_string)
    # span_word_pieces might be empty if the span consisted only of characters
    # discarded by the tokenizer.
    if span_word_pieces:
      word_pieces += span_word_pieces
      span_token_end = len(word_pieces) - 1
      token_spans.append((span_token_start, span_token_end))
      span_indexes.append(i)

  remaining_word_pieces = tokenizer.tokenize(sentence[last_index:])

  word_pieces += remaining_word_pieces

  last_token_idx = max_length - int(add_bert_tokens)
  if len(word_pieces) > last_token_idx:
    truncated_token_spans = [
        (start, end) for start, end in token_spans if end < last_token_idx
    ]
    num_dropped = len(token_spans) - len(truncated_token_spans)
    if num_dropped:
      logging.warn("Dropped %d spans due to length.", num_dropped)
      logging.warn("%d word pieces.", len(word_pieces))
      if not allow_truncated_spans:
        raise RuntimeError("Dropping spans not allowed when "
                           "allow_truncated_spans is False")
    word_pieces = word_pieces[:last_token_idx]
  else:
    truncated_token_spans = token_spans

  span_indexes = span_indexes[:len(truncated_token_spans)]

  if add_bert_tokens:
    word_pieces.append("[SEP]")
  word_piece_ids, word_piece_mask = convert_tokens_to_ids_and_pad(
      word_pieces, max_length, tokenizer)

  return (word_pieces, word_piece_ids, word_piece_mask, truncated_token_spans,
          span_indexes)
Tokenizes and resolves byte-offsets to word-offsets.

  Given a sentence and a set of inclusive byte-offset spans, applies a BERT
  tokenizer to the sentence and returns inclusive word-piece spans.

  Args:
    tokenizer: a BERT tokenizer
    sentence: a string, sentence to be tokenized
    spans: [first token, last token] byte-offset spans. The spans should be
      sorted by start_offset, and should also be non-overlapping.
    max_length: the maximum length to truncate or pad to.
    add_bert_tokens: whether to add special BERT tokens (CLS, SEP) to the input.
    allow_truncated_spans: when it is okay to drop spans due to length. This
      should be false for official evals!

  Returns:
    word_pieces: tokenized word pieces.
    word_piece_ids: IDs of tokenized word pieces.
    word_piece_mask: mask of word pieces (for padding).
    token_spans: inclusive word-piece spans of the input byte-offset spans.
    span_indexes: indexes with respect to the input spans corresponding to
      output token spans. This is necessary because some input spans can be
      truncated or otherwise dropped.

  Raises:
    RuntimeError: if allow_truncated_spans is False but spans are dropped due
      to length.
  
language/mentionmemory/utils/tokenization_utils.py