def process_syn_raw_text(data):
    """Process the output sequence from the instruction synthesizer"""
    read_collection_list = []
    for entry in data:
        cur_QA_list = parse_comprehension(entry['pred'], deduplicate = True)
        prev_read_collection = entry.get('prev_read_collection', [])
        # , 'pred': entry['pred']
        cur_entry = {'context': entry['text'], 'QA_list': cur_QA_list}

        if len(prev_read_collection) > 0:
            # check if the cur_entry successfully inherit from the latest prev_read_entry
            prev_entry = prev_read_collection[-1]

            if prev_entry['followed_by_next']:
                # followd_by_next ONLY indicates the prev_entry is prepended the cur_context when synthesizing the cur_entry
                # we need the 'inherit_from_prev' to indicates whether cur_entry indeed inherits from the prev_entry
                cur_entry['inherit_from_prev'] = True if len(cur_QA_list) > 0 else False
            else:
                # prev_entry is not presented when synthesizing the cur_entry
                cur_entry['inherit_from_prev'] = False
        else:
            cur_entry['inherit_from_prev'] = False

        # save the cur_entry to the 'prev_read_collection'
        prev_read_collection.append(cur_entry)
        read_collection_list.append(prev_read_collection)

    return read_collection_list
Process the output sequence from the instruction synthesizer
instruction_pretrain/utils/read_compre.py