def parse_comprehension(compre_str, deduplicate = False):
    """Get the list of QAs from the generated comprehension text"""
    QA_str_list = compre_str.split('</END>') # the list of `\n\n<QUE> {question} <ANS> {answer} `
    if not compre_str.endswith('</END>'):
        # This menas the last QA pair is incomplete, remove it
        QA_str_list = QA_str_list[:-1]

    QA_list = []
    for QA_str in QA_str_list:
        try:
            assert len(QA_str.split('<ANS>')) == 2, f'invalid QA string: {QA_str}'
            Q_str, A_str = QA_str.split('<ANS>') # `\n\n<QUE> {question} `, ` {answer}`
            Q_str, A_str = Q_str.strip(), A_str.strip() # `<QUE> {question}`, `{answer}`
            assert Q_str.startswith('<QUE>'), f'invalid question string: {Q_str} in QA_str: {QA_str}'
            assert len(A_str)>0, f'invalid answer string in QA_str: {QA_str}'
            Q_str = Q_str.replace('<QUE>', '').strip()
            QA_list.append({'Q': Q_str, 'A': A_str})
        except Exception as e:
            continue

    if deduplicate:
        # deduplicate questions
        QA_list = deduplicate_questions(QA_list)

    return QA_list
Get the list of QAs from the generated comprehension text
instruction_pretrain/utils/read_compre.py