def preprocess(
        self,
        qa_dataset,
        is_training,
        max_question_length=64,
        max_seq_length=MAX_SEQ_LEN,
        doc_stride=128,
        feature_cache_dir="./cached_qa_features",
    ):
        """
        Preprocesses raw question answering data and generates train/test features.

        Args:
            qa_dataset (:class:`utils_nlp.dataset.pytorch.QADataset`):
                Question answering data in standard QADataset format.
            is_training (bool): Whether the input data is training data.
            max_question_length (int, optional): Maximum number of tokens
                of the question sequence after tokenization, so the number of words
                in the raw question is usually less than max_question_length.
                Defaults to 64.
            max_seq_length (int, optional): Maximum number of tokens of the entire
                feature token sequence after tokenization. The entire feature token
                sequence is composed of:
                [CLS] + [Question tokens] + [SEP] + [Document tokens] + [SEP]
                for models other than XLNet,
                and [Document tokens] + [SEP] + [Question tokens] + [SEP] + [CLS} for
                XLNet. Defaults to MAX_SEQ_LEN.
            doc_stride (int, optional): Size (number of tokens) of the sliding window
                when breaking down a long document paragraph in to multiple document
                spans. Defaults to 128.
            feature_cache_dir (int, optional): Directory to save some intermediate
                preprocessing results.
                If `is_training` is True, CACHED_EXAMPLES_TRAIN_FILE and
                CACHED_FEATURES_TRAIN_FILE are saved to this directory. Otherwise,
                CACHED_EXAMPLES_TEST_FILE and CACHED_FEATURES_TEST_FILE are saved
                to this directory. These files are required during postprocessing to
                generate the final answer texts from predicted answer start and answer
                end indices. Defaults to "./cached_qa_features".
        Returns:
            DataSet: A Pytorch DataSet.
        """

        if not os.path.exists(feature_cache_dir):
            os.makedirs(feature_cache_dir)

        if is_training and not qa_dataset.actual_answer_available:
            raise Exception(
                "answer_start and answer_text must be provided for training data."
            )

        if is_training:
            examples_file = os.path.join(feature_cache_dir, CACHED_EXAMPLES_TRAIN_FILE)
            features_file = os.path.join(feature_cache_dir, CACHED_FEATURES_TRAIN_FILE)
        else:
            examples_file = os.path.join(feature_cache_dir, CACHED_EXAMPLES_TEST_FILE)
            features_file = os.path.join(feature_cache_dir, CACHED_FEATURES_TEST_FILE)

        with jsonlines.open(examples_file, "w") as examples_writer, jsonlines.open(
            features_file, "w"
        ) as features_writer:

            unique_id_all = []
            unique_id_cur = 1000000000

            features = []
            qa_examples = []
            qa_examples_json = []
            features_json = []

            for qa_input in qa_dataset:
                qa_example_cur = _create_qa_example(qa_input, is_training=is_training)

                qa_examples.append(qa_example_cur)

                qa_examples_json.append(
                    {
                        "qa_id": qa_example_cur.qa_id,
                        "doc_tokens": qa_example_cur.doc_tokens,
                    }
                )

                features_cur = _create_qa_features(
                    qa_example_cur,
                    model_type=self.model_type,
                    tokenizer=self.tokenizer,
                    unique_id=unique_id_cur,
                    is_training=is_training,
                    max_question_length=max_question_length,
                    max_seq_length=max_seq_length,
                    doc_stride=doc_stride,
                    custom_tokenize=self.custom_tokenize,
                )
                features += features_cur

                for f in features_cur:
                    features_json.append(
                        {
                            "qa_id": f.qa_id,
                            "unique_id": f.unique_id,
                            "tokens": f.tokens,
                            "token_to_orig_map": f.token_to_orig_map,
                            "token_is_max_context": f.token_is_max_context,
                            "paragraph_len": f.paragraph_len,
                        }
                    )
                    unique_id_cur = f.unique_id
                    unique_id_all.append(unique_id_cur)

            examples_writer.write_all(qa_examples_json)
            features_writer.write_all(features_json)

            logger.info("QA examples are saved to {}".format(examples_file))
            logger.info("QA features are saved to {}".format(features_file))

        # TODO: maybe generalize the following code
        input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long)
        input_mask = torch.tensor([f.input_mask for f in features], dtype=torch.long)
        segment_ids = torch.tensor([f.segment_ids for f in features], dtype=torch.long)
        cls_index = torch.tensor([f.cls_index for f in features], dtype=torch.long)
        p_mask = torch.tensor([f.p_mask for f in features], dtype=torch.long)

        if is_training:
            start_positions = torch.tensor(
                [f.start_position for f in features], dtype=torch.long
            )
            end_positions = torch.tensor(
                [f.end_position for f in features], dtype=torch.long
            )
            qa_dataset = TensorDataset(
                input_ids,
                input_mask,
                segment_ids,
                start_positions,
                end_positions,
                cls_index,
                p_mask,
            )
        else:
            unique_id_all = torch.tensor(unique_id_all, dtype=torch.long)
            qa_dataset = TensorDataset(
                input_ids, input_mask, segment_ids, cls_index, p_mask, unique_id_all
            )

        return qa_dataset

        Preprocesses raw question answering data and generates train/test features.

        Args:
            qa_dataset (:class:`utils_nlp.dataset.pytorch.QADataset`):
                Question answering data in standard QADataset format.
            is_training (bool): Whether the input data is training data.
            max_question_length (int, optional): Maximum number of tokens
                of the question sequence after tokenization, so the number of words
                in the raw question is usually less than max_question_length.
                Defaults to 64.
            max_seq_length (int, optional): Maximum number of tokens of the entire
                feature token sequence after tokenization. The entire feature token
                sequence is composed of:
                [CLS] + [Question tokens] + [SEP] + [Document tokens] + [SEP]
                for models other than XLNet,
                and [Document tokens] + [SEP] + [Question tokens] + [SEP] + [CLS} for
                XLNet. Defaults to MAX_SEQ_LEN.
            doc_stride (int, optional): Size (number of tokens) of the sliding window
                when breaking down a long document paragraph in to multiple document
                spans. Defaults to 128.
            feature_cache_dir (int, optional): Directory to save some intermediate
                preprocessing results.
                If `is_training` is True, CACHED_EXAMPLES_TRAIN_FILE and
                CACHED_FEATURES_TRAIN_FILE are saved to this directory. Otherwise,
                CACHED_EXAMPLES_TEST_FILE and CACHED_FEATURES_TEST_FILE are saved
                to this directory. These files are required during postprocessing to
                generate the final answer texts from predicted answer start and answer
                end indices. Defaults to "./cached_qa_features".
        Returns:
            DataSet: A Pytorch DataSet.
        
utils_nlp/models/transformers/question_answering.py