def preprocess_retriever_data(
    samples: List[Dict],
    gold_info_file: Optional[str],
    tensorizer: Tensorizer,
    cfg: ReaderPreprocessingCfg = DEFAULT_PREPROCESSING_CFG_TRAIN,
    is_train_set: bool = True,
) -> Iterable[ReaderSample]:
    """
    Converts retriever results into reader training data.
    :param samples: samples from the retriever's json file results
    :param gold_info_file: optional path for the 'gold passages & questions' file. Required to get best results for NQ
    :param tensorizer: Tensorizer object for text to model input tensors conversions
    :param cfg: ReaderPreprocessingCfg object with positive and negative passage selection parameters
    :param is_train_set: if the data should be processed as a train set
    :return: iterable of ReaderSample objects which can be consumed by the reader model
    """
    sep_tensor = tensorizer.get_pair_separator_ids()  # separator can be a multi token

    gold_passage_map, canonical_questions = (
        _get_gold_ctx_dict(gold_info_file) if gold_info_file else ({}, {})
    )

    no_positive_passages = 0
    positives_from_gold = 0

    def create_reader_sample_ids(sample: ReaderPassage, question: str):
        question_and_title = tensorizer.text_to_tensor(
            sample.title, title=question, add_special_tokens=True
        )
        if sample.passage_token_ids is None:
            sample.passage_token_ids = tensorizer.text_to_tensor(
                sample.passage_text, add_special_tokens=False
            )

        all_concatenated, shift = _concat_pair(
            question_and_title,
            sample.passage_token_ids,
            tailing_sep=sep_tensor if cfg.use_tailing_sep else None,
        )

        sample.sequence_ids = all_concatenated
        sample.passage_offset = shift
        assert shift > 1
        if sample.has_answer and is_train_set:
            sample.answers_spans = [
                (span[0] + shift, span[1] + shift) for span in sample.answers_spans
            ]
        return sample

    for sample in samples:
        question = sample["question"]

        if question in canonical_questions:
            question = canonical_questions[question]

        positive_passages, negative_passages = _select_reader_passages(
            sample,
            question,
            tensorizer,
            gold_passage_map,
            cfg.gold_page_only_positives,
            cfg.max_positives,
            cfg.max_negatives,
            cfg.min_negatives,
            cfg.max_retriever_passages,
            cfg.include_gold_passage,
            is_train_set,
        )
        # create concatenated sequence ids for each passage and adjust answer spans
        positive_passages = [
            create_reader_sample_ids(s, question) for s in positive_passages
        ]
        negative_passages = [
            create_reader_sample_ids(s, question) for s in negative_passages
        ]

        if is_train_set and len(positive_passages) == 0:
            no_positive_passages += 1
            if cfg.skip_no_positves:
                continue

        if next(iter(ctx for ctx in positive_passages if ctx.score == -1), None):
            positives_from_gold += 1

        if is_train_set:
            yield ReaderSample(
                question,
                sample["answers"],
                positive_passages=positive_passages,
                negative_passages=negative_passages,
            )
        else:
            yield ReaderSample(question, sample["answers"], passages=negative_passages)

    logger.info("no positive passages samples: %d", no_positive_passages)
    logger.info("positive passages from gold samples: %d", positives_from_gold)

    Converts retriever results into reader training data.
    :param samples: samples from the retriever's json file results
    :param gold_info_file: optional path for the 'gold passages & questions' file. Required to get best results for NQ
    :param tensorizer: Tensorizer object for text to model input tensors conversions
    :param cfg: ReaderPreprocessingCfg object with positive and negative passage selection parameters
    :param is_train_set: if the data should be processed as a train set
    :return: iterable of ReaderSample objects which can be consumed by the reader model
    
se2/DPR/dpr/data/reader_data.py