def generate_nodes_seq_and_write_to_file(compressed_args):
    """Extracts all the xpaths and labels the nodes for all the pages."""

    vertical, website = compressed_args

    all_data_dict = load_html_and_groundtruth(vertical, website)
    get_field_xpaths(
        all_data_dict,
        vertical_to_process=vertical,
        website_to_process=website,
        n_pages=2000,
        max_variable_nodes_per_website=300
    )
    """
    keys to the following example --->
        example = all_data_dict["0000"]

    dict_keys([
        'field-publication_date', 
        'field-author', 
        'field-title', 
        'field-publisher', 
        'field-isbn_13', 
        'html_str', 
        'path', 
        'dom_tree', 
        'xpath_data'
        ])
    """

    variable_nodes = all_data_dict["variable_nodes"]

    cleaned_features_for_this_website = {}

    for index in all_data_dict:
        if not index.isdigit():
            # Skip the cases when index is actually the "fixed/variable_nodes" keys.
            continue
        if int(index) >= FLAGS.n_pages:
            break
        page_data = all_data_dict[index]
        assert "xpath_data" in page_data

        doc_strings = page_data["doc_strings"]

        new_doc_strings = []

        field_info = {}
        for field in page_data:
            if not field.startswith("field-"):
                continue
            for doc_string_id in page_data[field]["is_truth_value_list"]:
                field_info[doc_string_id] = field[6:]

        for id, doc_string in enumerate(doc_strings):
            text, xpath = doc_string
            is_variable = xpath in variable_nodes
            if not is_variable:
                new_doc_strings.append((text, xpath, "fixed-node"))
            else:
                # for variable nodes,we need to give them labels
                gt_field = field_info.get(id, "none")
                new_doc_strings.append((text, xpath, gt_field))

        cleaned_features_for_this_website[index] = new_doc_strings

    output_file_path = os.path.join(FLAGS.output_data_path, f"{vertical}-{website}-{FLAGS.n_pages}.pickle")
    print(f"Writing the processed first {FLAGS.n_pages} pages of {vertical}-{website} into {output_file_path}")
    with open(output_file_path, "wb") as f:
        pickle.dump(cleaned_features_for_this_website, f)
Extracts all the xpaths and labels the nodes for all the pages.
markuplm/examples/fine_tuning/run_swde/prepare_data.py