def __init__(
      self,
      *,
      vocab_list,
      tokenizer,
      include_title,
      include_sentence_id,
      oov_buckets = 1,
      oov_token = 'UNK',
      lowercase = False,
      strip_vocab = True,
      decode_token_separator = ' ',
      max_claim_tokens = None,
      max_evidence_tokens = None):
    """Constructs a TokenTextEncoder.

    To load from a file saved with `TokenTextEncoder.save_to_file`, use
    `TokenTextEncoder.load_from_file`.

    Args:
      vocab_list: `list<str>`, list of tokens.
      tokenizer: `Tokenizer`, responsible for converting incoming text into a
        list of tokens.
      include_title: Whether to include wikipedia title in evidence
      include_sentence_id: Whether to include sentence_id in evidence
      oov_buckets: `int`, the number of `int`s to reserve for OOV hash buckets.
        Tokens that are OOV will be hash-modded into a OOV bucket in `encode`.
      oov_token: `str`, the string to use for OOV ids in `decode`.
      lowercase: `bool`, whether to make all text and tokens lowercase.
      strip_vocab: `bool`, whether to strip whitespace from the beginning and
        end of elements of `vocab_list`.
      decode_token_separator: `str`, the string used to separate tokens when
        decoding.
      max_claim_tokens: maximum number of claim tokens to use
      max_evidence_tokens: maximum number of evidence tokens to use
    """
    self._max_claim_tokens = max_claim_tokens
    self._max_evidence_tokens = max_evidence_tokens
    self._include_title = include_title
    self._include_sentence_id = include_sentence_id
    self._vocab_list = [tf.compat.as_text(el) for el in vocab_list]
    if strip_vocab:
      self._vocab_list = [el.strip() for el in self._vocab_list]
    self._lowercase = lowercase
    if self._lowercase:
      self._vocab_list = [t.lower() for t in self._vocab_list]
    # Note that internally everything is 0-indexed. Padding is dealt with at the
    # end of encode and the beginning of decode.
    self._token_to_id = dict(
        zip(self._vocab_list, range(len(self._vocab_list))))
    self._oov_buckets = oov_buckets
    self._oov_token = tf.compat.as_text(oov_token)

    self._tokenizer = tokenizer
    self._decode_token_separator = decode_token_separator
Constructs a TokenTextEncoder.

    To load from a file saved with `TokenTextEncoder.save_to_file`, use
    `TokenTextEncoder.load_from_file`.

    Args:
      vocab_list: `list<str>`, list of tokens.
      tokenizer: `Tokenizer`, responsible for converting incoming text into a
        list of tokens.
      include_title: Whether to include wikipedia title in evidence
      include_sentence_id: Whether to include sentence_id in evidence
      oov_buckets: `int`, the number of `int`s to reserve for OOV hash buckets.
        Tokens that are OOV will be hash-modded into a OOV bucket in `encode`.
      oov_token: `str`, the string to use for OOV ids in `decode`.
      lowercase: `bool`, whether to make all text and tokens lowercase.
      strip_vocab: `bool`, whether to strip whitespace from the beginning and
        end of elements of `vocab_list`.
      decode_token_separator: `str`, the string used to separate tokens when
        decoding.
      max_claim_tokens: maximum number of claim tokens to use
      max_evidence_tokens: maximum number of evidence tokens to use
    
language/serene/preprocessing.py