def basic_tokenize(text_input, lower_case=False, keep_whitespace=False):
  """Performs basic word tokenization for BERT.

  Args:
    text_input: A Tensor of untokenized strings.
    lower_case: A bool indicating whether or not to perform lowercasing. Default
      is False.
    keep_whitespace: A bool indicating whether or not whitespace tokens should
      be kept in the output
  """
  # lowercase and strip accents (if option is set)
  if lower_case:
    text_input = tf_text.case_fold_utf8(text_input)

  # normalize by NFD
  text_input = tf_text.normalize_utf8(text_input, "NFD")

  # strip out control characters
  text_input = tf.strings.regex_replace(text_input, r"\p{Cc}|\p{Cf}|\p{Mn}", "")

  # For chinese and emoji characters, tokenize by unicode codepoints
  script_tokenized = tf_text.unicode_script_tokenize(
      text_input, keep_whitespace=keep_whitespace, name="UTF-8")
  token_script_ids = tf.strings.unicode_script(
      tf.strings.unicode_decode(script_tokenized.flat_values, "UTF-8"))

  is_chinese = tf.equal(token_script_ids, _CHINESE_SCRIPT_ID)[:, :1].values
  is_emoji = tf_text.wordshape(script_tokenized.flat_values,
                               tf_text.WordShape.HAS_EMOJI)
  is_punct = tf_text.wordshape(script_tokenized.flat_values,
                               tf_text.WordShape.IS_PUNCT_OR_SYMBOL)
  split_cond = is_chinese | is_emoji | is_punct
  unicode_char_split = tf.strings.unicode_split(script_tokenized, "UTF-8")

  unicode_split_tokens = tf.where(
      split_cond,
      y=tf.expand_dims(script_tokenized.flat_values, 1),
      x=unicode_char_split.values)

  # Pack back into a [batch, (num_tokens), (num_unicode_chars)] RT
  chinese_mix_tokenized = tf.RaggedTensor.from_row_lengths(
      values=unicode_split_tokens, row_lengths=script_tokenized.row_lengths())

  # Squeeze out to a [batch, (num_tokens)] RT
  return collapse_dims(chinese_mix_tokenized)
Performs basic word tokenization for BERT.

  Args:
    text_input: A Tensor of untokenized strings.
    lower_case: A bool indicating whether or not to perform lowercasing. Default
      is False.
    keep_whitespace: A bool indicating whether or not whitespace tokens should
      be kept in the output
  
language/conpono/reconstruct/preprocess.py