examples/legacy/seq2seq/pack_dataset.py/pack_examples
def pack_examples(tok, src_examples, tgt_examples, max_tokens=1024):
    finished_src, finished_tgt = [], []

    sorted_examples = list(zip(src_examples, tgt_examples))
    new_src, new_tgt = sorted_examples[0]

    def is_too_big(strang):
        return tok(strang, return_tensors="pt").input_ids.shape[1] > max_tokens

    for src, tgt in tqdm(sorted_examples[1:]):
        cand_src = new_src + " " + src
        cand_tgt = new_tgt + " " + tgt
        if is_too_big(cand_src) or is_too_big(cand_tgt):  # cant fit, finalize example
            finished_src.append(new_src)
            finished_tgt.append(new_tgt)
            new_src, new_tgt = src, tgt
        else:  # can fit, keep adding
            new_src, new_tgt = cand_src, cand_tgt

    # cleanup
    if new_src:
        assert new_tgt
        finished_src.append(new_src)
        finished_tgt.append(new_tgt)
    return finished_src, finished_tgt