def build_thief_vocab(thief_paragraphs):
  """Return unigram freqs of thief dataset used for building random paras."""
  thief_vocab = cll.Counter("\n".join(thief_paragraphs).split())
  thief_vocab = [(k, v) for k, v in thief_vocab.items()]
  thief_vocab.sort(key=lambda x: x[1], reverse=True)
  thief_vocab_total = sum([x[1] for x in thief_vocab])

  thief_tokens = [x[0] for x in thief_vocab]
  thief_probs = [float(x[1]) / thief_vocab_total for x in thief_vocab]

  thief_lens = [len(x.split()) for x in thief_paragraphs]

  return thief_tokens, thief_probs, thief_lens
Return unigram freqs of thief dataset used for building random paras.
language/bert_extraction/steal_bert_qa/data_generation/preprocess_util.py