def tokenize(self, text):
    """Tokenize text.

    Args:
      text: text in natural language

    Returns:
      a list of tokens, segment_ids, and input mask
    """

    text = tokenization.convert_to_unicode(text)
    tokens = self.tokenizer.tokenize(text)
    tokens = ['[CLS]'] + tokens + ['[SEP]']
    token_ids = self.tokenizer.convert_tokens_to_ids(tokens)
    seg_ids = [0] * len(tokens)
    input_mask = [1] * len(tokens)
    assert self.max_seq_length > len(tokens)
    pad_len = self.max_seq_length - len(tokens)
    token_ids = np.array(token_ids + [0] * pad_len, dtype='int32')
    seg_ids = np.array(seg_ids + [0] * pad_len, dtype='int32')
    input_mask = np.array(input_mask + [0] * pad_len, dtype='int32')
    return tokens, (token_ids, seg_ids, input_mask)
Tokenize text.

    Args:
      text: text in natural language

    Returns:
      a list of tokens, segment_ids, and input mask
    
language/emql/util.py