def _build_vocab(self, dataset):
    """Build the vocabulary and encoder from the dataset.

    Args:
      dataset: The dataset to build vocab from.

    Returns:
      The vocabulary in the dataset, or empty vocab if using bert
    """
    # If we are using bert, then we do not need to build the vocab
    # since its already defined
    if self._model_config.tokenizer == 'bert' and self._model_config.text_encoder == 'bert':
      logging.info('Using bert, skipping vocabulary creation')
      return set()

    if self._tokenizer is None:
      raise ValueError('Cannot build vocab without a tokenizer.')
    claim_lengths = []
    evidence_lengths = []
    vocab = set()
    for example in tqdm.tqdm(dataset, mininterval=5):
      tokenized_claim, tokenized_evidence = self._tokenize_example(example)
      claim_lengths.append(len(tokenized_claim))
      evidence_lengths.append(len(tokenized_evidence))
      vocab.update(tokenized_claim)
      vocab.update(tokenized_evidence)
    logging.info('Build vocab of size (without padding): %s', len(vocab))
    logging.info('Claim length statistics')
    logging.info('Max: %s', max(claim_lengths))
    logging.info('Min: %s', min(claim_lengths))
    claim_percentiles = np.percentile(claim_lengths, [50, 90, 95, 99]).tolist()
    logging.info('50/90/95/99: %s', str(claim_percentiles))
    logging.info('Evidence length statistics')
    logging.info('Max: %s', max(evidence_lengths))
    logging.info('Min: %s', min(evidence_lengths))
    evidence_percentiles = np.percentile(evidence_lengths,
                                         [50, 90, 95, 99]).tolist()
    logging.info('50/90/95/99: %s', str(evidence_percentiles))
    self._vocab_stats['claim_max'] = max(claim_lengths)
    self._vocab_stats['claim_min'] = min(claim_lengths)
    self._vocab_stats['claim_percentiles'] = claim_percentiles
    self._vocab_stats['evidence_max'] = max(evidence_lengths)
    self._vocab_stats['evidence_min'] = min(evidence_lengths)
    self._vocab_stats['evidence_percentiles'] = evidence_percentiles
    return vocab
Build the vocabulary and encoder from the dataset.

    Args:
      dataset: The dataset to build vocab from.

    Returns:
      The vocabulary in the dataset, or empty vocab if using bert
    
language/serene/training.py