def wordpiece_tokenize(text_input,
                       vocab_table,
                       token_out_type=tf.int64,
                       use_unknown_token=True,
                       lower_case=False,
                       keep_whitespace=False):
  """Perform wordpiece tokenization on a Tensor of untokenized strings.

  Args:
    text_input: <tf.string> A Tensor of untokenized strings.
    vocab_table: A LookupInterface containing the vocabulary.
    token_out_type: the dtype desired as the output, default is int64.
    use_unknown_token: (optional) A bool indicating whether to return [UNK]
      token or leave out of vocabulary words as is. Default is True.
    lower_case: (optional) A bool indicating whether or not to perform
      lowercasing. Default is False.
    keep_whitespace: A bool indicating whether or not whitespace tokens should
      be kept in the output

  Returns:
    a RaggedTensor w/ shape [num_batch, (num_tokens), (num_wordpieces)].
  """
  tokenized = basic_tokenize(text_input, lower_case, keep_whitespace)

  subwords, _, _ = tf_text.wordpiece_tokenize_with_offsets(
      tokenized,
      vocab_table,
      suffix_indicator="##",
      use_unknown_token=use_unknown_token,
      token_out_type=token_out_type,
  )
  return subwords
Perform wordpiece tokenization on a Tensor of untokenized strings.

  Args:
    text_input: <tf.string> A Tensor of untokenized strings.
    vocab_table: A LookupInterface containing the vocabulary.
    token_out_type: the dtype desired as the output, default is int64.
    use_unknown_token: (optional) A bool indicating whether to return [UNK]
      token or leave out of vocabulary words as is. Default is True.
    lower_case: (optional) A bool indicating whether or not to perform
      lowercasing. Default is False.
    keep_whitespace: A bool indicating whether or not whitespace tokens should
      be kept in the output

  Returns:
    a RaggedTensor w/ shape [num_batch, (num_tokens), (num_wordpieces)].
  
language/conpono/reconstruct/preprocess.py