def build_question_starter_vocab_boolq(sents_data):
  """Build a vocab of first word  question tokens."""
  starter_vocab = []

  for instance in sents_data:
    question = instance["question"]
    starter_vocab.append(question.split()[0])

  starter_vocab = cll.Counter(starter_vocab)
  starter_vocab = [(k, v) for k, v in starter_vocab.items() if v >= 50]
  starter_vocab.sort(key=lambda x: x[1], reverse=True)
  start_vocab_total = sum([x[1] for x in starter_vocab])

  starter_vocab_tokens = [x[0] for x in starter_vocab]
  starter_vocab_probs = [float(x[1]) / start_vocab_total for x in starter_vocab]

  return starter_vocab_tokens, starter_vocab_probs
Build a vocab of first word  question tokens.
language/bert_extraction/steal_bert_qa/data_generation/preprocess_util.py