def bert_preprocess(text_segments,
                    vocab_table,
                    max_seq_length,
                    max_predictions_per_seq,
                    tokenize_fn=None,
                    mask_rate=0.15,
                    do_lower_case=True,
                    cls_token=_CLS_TOKEN,
                    sep_token=_SEP_TOKEN,
                    mask_token=_MASK_TOKEN):
  """Pre-processes a text tuple into BERT format.

  Args:
    text_segments (Tensor): 1-D Tensor of un-tokenized text segments (string).
      These will be concatenated into a single sequence.
    vocab_table (StaticHashTable): a map from string to integer.
    max_seq_length (int): max # of wordpiece tokens, including special tokens.
    max_predictions_per_seq (int): max # of masked positions for entire seq.
    tokenize_fn: a function that performs a tokenization on a 1D string Tensor
      of untokenized strings. By default this uses wordpiece tokenization.
    mask_rate (float): percentage of tokens to mask out. If 0, no masking is
      performed.
    do_lower_case (bool): whether to lowercase text or not. Default is True.
    cls_token (unicode): token representing CLS
    sep_token (unicode): token representing SEP (separator)
    mask_token (unicode): token representing MASK

  Returns:
    BertInputs
  """

  if not tokenize_fn:
    # pylint: disable=g-long-lambda
    tokenize_fn = lambda text_input: wordpiece_tokenize(
        text_input=text_segments,
        vocab_table=vocab_table,
        token_out_type=tf.string,
        use_unknown_token=True,
        lower_case=do_lower_case)

  segment_tokens_grouped = tokenize_fn(text_segments)
  # TODO(kguu): check that BERT handles UNK in the same way.

  # This is a RaggedTensor of shape [batch_size, (num_wordpieces)]
  # One row for each segment. Each row is a list of wordpiece tokens.
  segment_tokens = collapse_dims(segment_tokens_grouped)

  # Truncate.
  num_special_tokens = segment_tokens.nrows() + 1
  segment_tokens_truncated = truncate_segment_tokens(
      segment_tokens, max_seq_length - num_special_tokens)

  # Add special tokens.
  segment_tokens_with_special_tokens = add_special_tokens(
      segment_tokens_truncated, cls_token, sep_token)

  # Compute segment IDs.
  segment_ids_2d = create_segment_ids(segment_tokens_with_special_tokens)

  # Flatten everything to a 1-D string Tensor.
  tokens = segment_tokens_with_special_tokens.flat_values
  segment_ids = segment_ids_2d.flat_values

  # Convert to token IDs.
  token_ids = vocab_table.lookup(tokens)

  # Apply masking.
  mask_blacklist = tf.constant([cls_token, sep_token])
  mask_blacklist_ids = vocab_table.lookup(mask_blacklist)
  mask_token_id = vocab_table.lookup(tf.constant(mask_token))

  mask_indices = sample_mask_indices(token_ids, mask_rate, mask_blacklist_ids,
                                     max_predictions_per_seq)

  target_token_ids = get_target_tokens_for_apply(token_ids, mask_indices)
  token_ids_masked = apply_masking(token_ids, target_token_ids, mask_indices,
                                   mask_token_id, vocab_table.size())

  pad_mask = tf.ones_like(token_ids_masked)
  target_token_weights = tf.ones_like(target_token_ids, dtype=tf.float32)

  # Pad to max_seq_length.
  token_ids_masked = pad_to_length(token_ids_masked, max_seq_length)
  pad_mask = pad_to_length(pad_mask, max_seq_length)
  segment_ids = pad_to_length(segment_ids, max_seq_length)

  # Pad to max_predictions_per_seq.
  mask_indices = pad_to_length(mask_indices, max_predictions_per_seq)
  target_token_ids = pad_to_length(target_token_ids, max_predictions_per_seq)
  target_token_weights = pad_to_length(target_token_weights,
                                       max_predictions_per_seq)

  return BertInputs(
      input_ids=token_ids_masked,
      input_mask=pad_mask,
      segment_ids=segment_ids,
      masked_lm_positions=mask_indices,
      masked_lm_ids=target_token_ids,
      masked_lm_weights=target_token_weights)
Pre-processes a text tuple into BERT format.

  Args:
    text_segments (Tensor): 1-D Tensor of un-tokenized text segments (string).
      These will be concatenated into a single sequence.
    vocab_table (StaticHashTable): a map from string to integer.
    max_seq_length (int): max # of wordpiece tokens, including special tokens.
    max_predictions_per_seq (int): max # of masked positions for entire seq.
    tokenize_fn: a function that performs a tokenization on a 1D string Tensor
      of untokenized strings. By default this uses wordpiece tokenization.
    mask_rate (float): percentage of tokens to mask out. If 0, no masking is
      performed.
    do_lower_case (bool): whether to lowercase text or not. Default is True.
    cls_token (unicode): token representing CLS
    sep_token (unicode): token representing SEP (separator)
    mask_token (unicode): token representing MASK

  Returns:
    BertInputs
  
language/conpono/reconstruct/preprocess.py