def read_squad_examples(args, input_file, root_dir, is_training, tokenizer, simplify=False, max_depth=50,
                        split_flag="n-eon",
                        attention_width=None):
    r"""
    pre-process the data in json format into SRC Examples.

    Arguments:
        split_flag:
        attention_width:
        input_file (str): the inputting data file in json format.
        root_dir (str): the root directory of the raw WebSRC dataset, which contains the HTML files.
        is_training (bool): True if processing the training set, else False.
        tokenizer (Tokenizer): the tokenizer for PLM in use.
        method (str): the name of the method in use, choice: ['T-PLM', 'H-PLM', 'V-PLM'].
        simplify (bool): when setting to Ture, the returned Example will only contain document tokens, the id of the
                         question-answers, and the total tag number in the corresponding html files.
    Returns:
        list[SRCExamples]: the resulting SRC Examples, contained all the needed information for the feature generation
                           process, except when the argument simplify is setting to True;
        set[str]: all the tag names appeared in the processed dataset, e.g. <div>, <img/>, </p>, etc..
    """
    with open(input_file, "r", encoding='utf-8') as reader:
        input_data = json.load(reader)["data"]

    pad_tree_id_seq = [1001] * max_depth

    def is_whitespace(c):
        if c == " " or c == "\t" or c == "\r" or c == "\n" or ord(c) == 0x202F:
            return True
        return False

    def html_to_text_list(h):
        tag_num, text_list = 0, []
        for element in h.descendants:
            if (type(element) == bs4.element.NavigableString) and (element.strip()):
                text_list.append(element.strip())
            if type(element) == bs4.element.Tag:
                tag_num += 1
        return text_list, tag_num + 2  # + 2 because we treat the additional 'yes' and 'no' as two special tags.

    def html_to_text(h):
        tag_list = set()
        for element in h.descendants:
            if type(element) == bs4.element.Tag:
                element.attrs = {}
                temp = str(element).split()
                tag_list.add(temp[0])
                tag_list.add(temp[-1])
        return html_escape(str(h)), tag_list

    def adjust_offset(offset, text):
        text_list = text.split()
        cnt, adjustment = 0, []
        for t in text_list:
            if not t:
                continue
            if t[0] == '<' and t[-1] == '>':
                adjustment.append(offset.index(cnt))
            else:
                cnt += 1
        add = 0
        adjustment.append(len(offset))
        for i in range(len(offset)):
            while i >= adjustment[add]:
                add += 1
            offset[i] += add
        return offset

    def e_id_to_t_id(e_id, html):
        t_id = 0
        for element in html.descendants:
            if type(element) == bs4.element.NavigableString and element.strip():
                t_id += 1
            if type(element) == bs4.element.Tag:
                if int(element.attrs['tid']) == e_id:
                    break
        return t_id

    def calc_num_from_raw_text_list(t_id, l):
        n_char = 0
        for i in range(t_id):
            n_char += len(l[i]) + 1
        return n_char

    def word_to_tag_from_text(tokens, h):
        cnt, w_t, path = -1, [], []
        unique_tids = set()
        for t in tokens[0:-2]:
            if len(t) < 2:
                w_t.append(path[-1])
                unique_tids.add(path[-1])
                continue
            if t[0] == '<' and t[-2] == '/':
                cnt += 1
                w_t.append(cnt)
                unique_tids.add(cnt)
                continue
            if t[0] == '<' and t[1] != '/':
                cnt += 1
                path.append(cnt)
            w_t.append(path[-1])
            unique_tids.add(path[-1])
            if t[0] == '<' and t[1] == '/':
                del path[-1]
        w_t.append(cnt + 1)
        unique_tids.add(cnt + 1)
        w_t.append(cnt + 2)
        unique_tids.add(cnt + 2)
        assert len(w_t) == len(tokens)
        assert len(path) == 0, print(h)
        return w_t, unique_tids

    def word_tag_offset(html):
        cnt, w_t, t_w, tags, tags_tids = 0, [], [], [], []
        for element in html.descendants:
            if type(element) == bs4.element.Tag:
                content = ' '.join(list(element.strings)).split()
                t_w.append({'start': cnt, 'len': len(content)})
                tags.append('<' + element.name + '>')
                tags_tids.append(element['tid'])
            elif type(element) == bs4.element.NavigableString and element.strip():
                text = element.split()
                tid = element.parent['tid']
                ind = tags_tids.index(tid)
                for _ in text:
                    w_t.append(ind)
                    cnt += 1
                assert cnt == len(w_t)
        w_t.append(len(t_w))
        w_t.append(len(t_w) + 1)
        return w_t

    def subtoken_tag_offset(html, s_tok):
        w_t = word_tag_offset(html)
        s_t = []
        unique_tids = set()
        for i in range(len(s_tok)):
            s_t.append(w_t[s_tok[i]])
            unique_tids.add(w_t[s_tok[i]])
        return s_t, unique_tids

    def subtoken_tag_offset_plus_eon(html, s_tok, all_doc_tokens):
        w_t = word_tag_offset(html)
        s_t = []
        unique_tids = set()
        offset = 0
        for i in range(len(s_tok)):
            if all_doc_tokens[i] not in ('<end-of-node>', tokenizer.sep_token, tokenizer.cls_token):
                s_t.append(w_t[s_tok[i] - offset])
                unique_tids.add(w_t[s_tok[i] - offset])
            else:
                prev_tid = s_t[-1]
                s_t.append(prev_tid)
                offset += 1
        return s_t, unique_tids

    def check_visible(path1, path2, attention_width):
        i = 0
        j = 0
        dis = 0
        lp1 = len(path1)
        lp2 = len(path2)
        while i < lp1 and j < lp2 and path1[i] == path2[j]:
            i += 1
            j += 1

        if i < lp1 and j < lp2:
            dis += lp1 - i + lp2 - j
        else:
            if i == lp1:
                dis += lp2 - j
            else:
                dis += lp1 - i

        if dis <= attention_width:
            return True
        return False


    def from_tids_to_box(html_fn, unique_tids, json_fn):
        sorted_ids = sorted(unique_tids)
        f = open(json_fn, 'r')
        data = json.load(f)
        orig_width, orig_height = data['2']['rect']['width'], data['2']['rect']['height']
        orig_x, orig_y = data['2']['rect']['x'], data['2']['rect']['y']

        return_dict = {}
        for id in sorted_ids:
            if str(id) in data:
                x, y, width, height = data[str(id)]['rect']['x'], data[str(id)]['rect']['y'], data[str(id)]['rect']['width'], data[str(id)]['rect']['height']
                resize_x = (x - orig_x) * 1000 // orig_width
                resize_y = (y - orig_y) * 1000 // orig_height
                
                resize_width = width * 1000 // orig_width
                resize_height = height * 1000 // orig_height

                # if not (resize_x <= 1000 and resize_y <= 1000):
                #     print('before', x, y, width, height)
                #     print('after', resize_x, resize_y, resize_width, resize_height)
                #     print('file name ', html_fn)
                #     # exit(0)

                if resize_x < 0 or resize_y < 0 or resize_width < 0 or resize_height < 0: # meaningless
                    return_dict[id] = [0, 0, 0, 0]
                else:
                    return_dict[id] = [int(resize_x), int(resize_y), int(resize_x+resize_width), int(resize_y+resize_height)]
            else:
                return_dict[id] = [0,0,0,0]
       
        return return_dict

    def get_visible_matrix(unique_tids, tree_id_map, attention_width):
        if attention_width is None:
            return None
        unique_tids_list = list(unique_tids)
        visible_matrix = collections.defaultdict(list)
        for i in range(len(unique_tids_list)):
            if tree_id_map[unique_tids_list[i]] == pad_tree_id_seq:
                visible_matrix[unique_tids_list[i]] = list()
                continue
            visible_matrix[unique_tids_list[i]].append(unique_tids_list[i])
            for j in range(i + 1, len(unique_tids_list)):
                if check_visible(tree_id_map[unique_tids_list[i]], tree_id_map[unique_tids_list[j]], attention_width):
                    visible_matrix[unique_tids_list[i]].append(unique_tids_list[j])
                    visible_matrix[unique_tids_list[j]].append(unique_tids_list[i])
        return visible_matrix

    examples = []
    all_tag_list = set()
    total_num = sum([len(entry["websites"]) for entry in input_data])
    with tqdm(total=total_num, desc="Converting websites to examples") as t:
        for entry in input_data:
            # print('entry', entry)
            domain = entry["domain"]
            for website in entry["websites"]:
                # print('website', website)

                # Generate Doc Tokens
                page_id = website["page_id"]
                # print('page_id', page_id)
                curr_dir = osp.join(root_dir, domain, page_id[0:2], 'processed_data')
                html_fn = osp.join(curr_dir, page_id + '.html')
                json_fn = osp.join(curr_dir, page_id + '.json')

                # print('html', html_fn)

                html_file = open(html_fn).read()
                html_code = bs(html_file, "html.parser")
                raw_text_list, tag_num = html_to_text_list(html_code) 

                # print(raw_text_list)
                # print(tag_num)
                # exit(0)
                doc_tokens = []
                char_to_word_offset = []

                # print(split_flag) # n-eon
                # exit(0)

                if split_flag in ["y-eon", "y-sep", "y-cls"]:
                    prev_is_whitespace = True
                    for i, doc_string in enumerate(raw_text_list):
                        for c in doc_string:
                            if is_whitespace(c):
                                prev_is_whitespace = True
                            else:
                                if prev_is_whitespace:
                                    doc_tokens.append(c)
                                else:
                                    doc_tokens[-1] += c
                                prev_is_whitespace = False
                            char_to_word_offset.append(len(doc_tokens) - 1)

                        if i < len(raw_text_list) - 1:
                            prev_is_whitespace = True
                            char_to_word_offset.append(len(doc_tokens) - 1)

                        if split_flag == "y-eon":
                            doc_tokens.append('<end-of-node>')
                        elif split_flag == "y-sep":
                            doc_tokens.append(tokenizer.sep_token)
                        elif split_flag == "y-cls":
                            doc_tokens.append(tokenizer.cls_token)
                        else:
                            raise ValueError("Split flag should be `y-eon` or `y-sep` or `y-cls`")
                        prev_is_whitespace = True

                elif split_flag =="n-eon" or split_flag == "y-hplm":
                    page_text = ' '.join(raw_text_list) 
                    prev_is_whitespace = True
                    for c in page_text:
                        if is_whitespace(c):
                            prev_is_whitespace = True
                        else:
                            if prev_is_whitespace:
                                doc_tokens.append(c)
                            else:
                                doc_tokens[-1] += c
                            prev_is_whitespace = False
                        char_to_word_offset.append(len(doc_tokens) - 1) 


                doc_tokens.append('no')
                char_to_word_offset.append(len(doc_tokens) - 1)
                doc_tokens.append('yes')
                char_to_word_offset.append(len(doc_tokens) - 1)

                if split_flag == "y-hplm":
                    real_text, tag_list = html_to_text(bs(html_file))
                    all_tag_list = all_tag_list | tag_list
                    char_to_word_offset = adjust_offset(char_to_word_offset, real_text)
                    doc_tokens = real_text.split()
                    doc_tokens.append('no')
                    doc_tokens.append('yes')
                    doc_tokens = [i for i in doc_tokens if i]

                else:
                    tag_list = []

                assert len(doc_tokens) == char_to_word_offset[-1] + 1, (len(doc_tokens), char_to_word_offset[-1])

                if simplify:
                    for qa in website["qas"]:
                        qas_id = qa["id"]
                        example = SRCExample(doc_tokens=doc_tokens, qas_id=qas_id, tag_num=tag_num)
                        examples.append(example)
                    t.update(1)
                else:
                    # Tokenize all doc tokens
                    # tokenize sth like < / >
                    tok_to_orig_index = []
                    orig_to_tok_index = []
                    all_doc_tokens = []
                    for (i, token) in enumerate(doc_tokens):
                        orig_to_tok_index.append(len(all_doc_tokens))
                        if token in tag_list:
                            sub_tokens = [token]
                        else:
                            sub_tokens = tokenizer.tokenize(token)
                        for sub_token in sub_tokens:
                            tok_to_orig_index.append(i)
                            all_doc_tokens.append(sub_token)

                    # Generate extra information for features
                    if split_flag in ["y-eon", "y-sep", "y-cls"]:
                        tok_to_tags_index, unique_tids = subtoken_tag_offset_plus_eon(html_code, tok_to_orig_index,
                                                                                      all_doc_tokens)
                    elif split_flag == "n-eon":
                        tok_to_tags_index, unique_tids = subtoken_tag_offset(html_code, tok_to_orig_index)

                    elif split_flag == "y-hplm":
                        tok_to_tags_index, unique_tids = word_to_tag_from_text(all_doc_tokens, html_code)

                    else:
                        raise ValueError("Unsupported split_flag!")

                

                    xpath_tag_map, xpath_subs_map, tree_id_map = get_xpath_and_treeid4tokens(html_code, unique_tids,
                                                                                             max_depth=max_depth)
                    # tree_id_map : neither truncated nor padded
                    xpath_box = from_tids_to_box(html_fn, unique_tids, json_fn) 


                    assert tok_to_tags_index[-1] == tag_num - 1, (tok_to_tags_index[-1], tag_num - 1)

                    # we get attention_mask here
                    visible_matrix = get_visible_matrix(unique_tids, tree_id_map, attention_width=attention_width)

                    # Process each qas, which is mainly calculate the answer position
                    for qa in website["qas"]:
                        qas_id = qa["id"]
                        question_text = qa["question"]
                        start_position = None
                        end_position = None
                        orig_answer_text = None

                        if is_training:
                            if len(qa["answers"]) != 1:
                                raise ValueError(
                                    "For training, each question should have exactly 1 answer.")
                            answer = qa["answers"][0]
                            orig_answer_text = answer["text"]
                            if answer["element_id"] == -1:
                                num_char = len(char_to_word_offset) - 2
                            else:
                                num_char = calc_num_from_raw_text_list(e_id_to_t_id(answer["element_id"], html_code),
                                                                       raw_text_list)
                            answer_offset = num_char + answer["answer_start"]
                            answer_length = len(orig_answer_text) if answer["element_id"] != -1 else 1
                            start_position = char_to_word_offset[answer_offset]
                            end_position = char_to_word_offset[answer_offset + answer_length - 1]
                            # Only add answers where the text can be exactly recovered from the
                            # document. If this CAN'T happen it's likely due to weird Unicode
                            # stuff so we will just skip the example.
                            #
                            # Note that this means for training mode, every example is NOT
                            # guaranteed to be preserved.
                            actual_text = " ".join([w for w in doc_tokens[start_position:(end_position + 1)]
                                                    if (w[0] != '<' or w[-1] != '>')
                                                    and w != "<end-of-node>"
                                                    and w != tokenizer.sep_token
                                                    and w != tokenizer.cls_token])
                            cleaned_answer_text = " ".join(whitespace_tokenize(orig_answer_text))
                            if actual_text.find(cleaned_answer_text) == -1:
                                logging.warning("Could not find answer of question %s: '%s' vs. '%s'",
                                               qa['id'], actual_text, cleaned_answer_text)
                                continue

                        example = SRCExample(
                            doc_tokens=doc_tokens,
                            qas_id=qas_id,
                            tag_num=tag_num,
                            question_text=question_text,
                            html_code=html_code,
                            orig_answer_text=orig_answer_text,
                            start_position=start_position,
                            end_position=end_position,
                            tok_to_orig_index=tok_to_orig_index,
                            orig_to_tok_index=orig_to_tok_index,
                            all_doc_tokens=all_doc_tokens,
                            tok_to_tags_index=tok_to_tags_index,
                            xpath_tag_map=xpath_tag_map,
                            xpath_subs_map=xpath_subs_map,
                            xpath_box=xpath_box, 
                            tree_id_map=tree_id_map,
                            visible_matrix=visible_matrix
                        )

                        examples.append(example)

                        
                        if args.web_num_features != 0:
                            if len(examples) >= args.web_num_features:
                                return examples, all_tag_list

                    t.update(1)
    return examples, all_tag_list

    pre-process the data in json format into SRC Examples.

    Arguments:
        split_flag:
        attention_width:
        input_file (str): the inputting data file in json format.
        root_dir (str): the root directory of the raw WebSRC dataset, which contains the HTML files.
        is_training (bool): True if processing the training set, else False.
        tokenizer (Tokenizer): the tokenizer for PLM in use.
        method (str): the name of the method in use, choice: ['T-PLM', 'H-PLM', 'V-PLM'].
        simplify (bool): when setting to Ture, the returned Example will only contain document tokens, the id of the
                         question-answers, and the total tag number in the corresponding html files.
    Returns:
        list[SRCExamples]: the resulting SRC Examples, contained all the needed information for the feature generation
                           process, except when the argument simplify is setting to True;
        set[str]: all the tag names appeared in the processed dataset, e.g. <div>, <img/>, </p>, etc..
    
xdoc/fine_tuning/websrc/websrc.py