def convert_tokens_to_ids_and_pad(
    word_pieces: List[str], max_length: int,
    token_converter: TokenConverter) -> Tuple[List[int], List[int]]:
  """Converts the input tokens to padded ids and mask.

  Args:
    word_pieces: list of word-piece tokens.
    max_length: length sequences should be padded/truncated to.
    token_converter: the tokenizer used.

  Returns:

    word_piece_ids: IDs of wordpieces, padded or truncated to max_length.
    word_piece_mask: a mask for the wordpieces.

  """
  word_piece_ids = token_converter.convert_tokens_to_ids(word_pieces)
  word_piece_mask = [1] * len(word_pieces)

  # Retrieve pad token id from tokenizer. The method expects a list, so we pass
  # a list with a single token ("[PAD]") and  retrieve the first id returned.
  pad_id = token_converter.convert_tokens_to_ids(["[PAD]"])[0]
  while len(word_pieces) < max_length:
    word_pieces.append("")
    word_piece_ids.append(pad_id)
    word_piece_mask.append(0)

  return word_piece_ids, word_piece_mask
Converts the input tokens to padded ids and mask.

  Args:
    word_pieces: list of word-piece tokens.
    max_length: length sequences should be padded/truncated to.
    token_converter: the tokenizer used.

  Returns:

    word_piece_ids: IDs of wordpieces, padded or truncated to max_length.
    word_piece_mask: a mask for the wordpieces.

  
language/mentionmemory/utils/tokenization_utils.py