def build_text_inputs(
    text,
    length,
    lookup_table,
    segment_id=0,
    start_token=None,
    end_token=None,
):
  """Convert text to TextInputs.

  Args:
    text: <string>, space-separated token string.
    length: Length to pad or truncate to.
    lookup_table: Instance of contrib.lookup.index_table_from_tensor.
    segment_id: Integer denoting segment type.
    start_token: Optional start token.
    end_token: Optional end token.

  Returns:
    Instance of TextInputs.
  """
  # Tokenize and truncate.
  tokens = tf.string_split([text]).values
  length_offset = sum([0 if i is None else 1 for i in [start_token, end_token]])
  tokens = tokens[:length - length_offset]
  if start_token is not None:
    tokens = tf.concat([[start_token], tokens], axis=0)
  if end_token is not None:
    tokens = tf.concat([tokens, [end_token]], axis=0)

  token_ids = tf.cast(lookup_table.lookup(tokens), tf.int32)
  mask = tf.ones_like(token_ids)
  segment_ids = tf.fill(tf.shape(token_ids), segment_id)

  pad = [[0, length - tf.size(token_ids)]]
  token_ids = tf.pad(token_ids, pad)
  mask = tf.pad(mask, pad)
  segment_ids = tf.pad(segment_ids, pad)
  positions = tf.range(length)
  text_input = TextInputs(
      token_ids=tf.ensure_shape(token_ids, [length]),
      mask=tf.ensure_shape(mask, [length]),
      segment_ids=tf.ensure_shape(segment_ids, [length]),
      positions=tf.ensure_shape(positions, [length]))

  return text_input
Convert text to TextInputs.

  Args:
    text: <string>, space-separated token string.
    length: Length to pad or truncate to.
    lookup_table: Instance of contrib.lookup.index_table_from_tensor.
    segment_id: Integer denoting segment type.
    start_token: Optional start token.
    end_token: Optional end token.

  Returns:
    Instance of TextInputs.
  
language/capwap/utils/text_utils.py