def add_special_tokens(segment_tokens, cls_token, sep_token):
  """Adds special tokens to segment tokens.

  Appends a [SEP] token to each segment.
  Prepends a [CLS] token to the first segment.

  Args:
    segment_tokens (RaggedTensor): a 2-D RaggedTensor of strings. One row for
      each segment. Each row is a list of tokens.
    cls_token (unicode): string for CLS token.
    sep_token (unicode): string for SEP token.

  Returns:
    segment_tokens (Tensor): a 2-D string Tensor.
  """
  num_rows = tf.to_int32(segment_tokens.nrows())

  # One SEP token for every row.
  sep_tokens = tf.fill([num_rows, 1], sep_token)

  # One CLS token in the first row.
  cls_tokens = tf.RaggedTensor.from_row_lengths([cls_token],
                                                row_lengths=tf.one_hot(
                                                    0, num_rows,
                                                    dtype=tf.int64))

  segment_tokens = tf.concat([cls_tokens, segment_tokens, sep_tokens], axis=1)
  return segment_tokens
Adds special tokens to segment tokens.

  Appends a [SEP] token to each segment.
  Prepends a [CLS] token to the first segment.

  Args:
    segment_tokens (RaggedTensor): a 2-D RaggedTensor of strings. One row for
      each segment. Each row is a list of tokens.
    cls_token (unicode): string for CLS token.
    sep_token (unicode): string for SEP token.

  Returns:
    segment_tokens (Tensor): a 2-D string Tensor.
  
language/conpono/reconstruct/preprocess.py