def _build_megatron_dataset_splits(
        self, path_prefix: str, split: List[float], sizes: List[int],
    ) -> List[Optional[MegatronDataset]]:
        """Build each MegatronDataset split from a single MMapIndexedDataset

        Args:
            path_prefix (str): The MMapIndexedDataset .bin and .idx file prefix

            split (List[Tuple[float, float]]): The dataset split matrix

            sizes (List[int]): The number of total samples to draw from each split

        Returns:
            List[Optional[MegatronDataset]]: The MegatronDatset (or None) per split
        """
        indexed_dataset = self.build_generic_dataset(
            MMapIndexedDataset, self.config.is_built_on_rank, path_prefix, self.cls.is_multimodal(),
        )

        if indexed_dataset is not None:
            if self.cls.is_split_by_sequence():
                num_elements = indexed_dataset.sequence_lengths.shape[0]
            else:
                num_elements = indexed_dataset.document_indices.shape[0] - 1

            split_indices = []
            for i, _ in enumerate(Split):
                if split[i] is not None:
                    beg = int(round(split[i][0] * float(num_elements)))
                    end = int(round(split[i][1] * float(num_elements)))
                    split_indices.append(
                        numpy.arange(start=beg, stop=end, step=1, dtype=numpy.int32)
                    )
                else:
                    split_indices.append(None)
        else:
            split_indices = [None for _ in Split]

        megatron_datasets = []
        for i, _split in enumerate(Split):
            if split[i] is None:
                megatron_datasets.append(None)
            else:
                megatron_datasets.append(
                    self.build_generic_dataset(
                        self.cls,
                        self.config.is_built_on_rank,
                        indexed_dataset,
                        split_indices[i],
                        sizes[i],
                        _split,
                        self.config,
                    )
                )

        return megatron_datasets
Build each MegatronDataset split from a single MMapIndexedDataset

        Args:
            path_prefix (str): The MMapIndexedDataset .bin and .idx file prefix

            split (List[Tuple[float, float]]): The dataset split matrix

            sizes (List[int]): The number of total samples to draw from each split

        Returns:
            List[Optional[MegatronDataset]]: The MegatronDatset (or None) per split
        
finetune/core/datasets/blended_megatron_dataset_builder.py