def __init__(self, sos_token='<SOS>', eos_token='<EOS>', pad_token='<PAD>'):
    """NB: <PAD> token is by construction idx 0."""
    self.sos_token = sos_token
    self.eos_token = eos_token
    self.pad_token = pad_token
    self.idx2word = [pad_token, sos_token, eos_token]
    self.word2idx = {}
    self.word2idx[pad_token] = 0
    self.word2idx[sos_token] = 1
    self.word2idx[eos_token] = 2
    self.word_frequencies = collections.Counter()
NB: <PAD> token is by construction idx 0.
language/gscan/xattn_model/dataset/preprocess.py