def parse_files_and_find_duplicates(self, start_end_epochs, list_of_files, parent_dir):
        """Fetch the start and end epoch files data, get the file names with potential duplicates and remove duplicates.

        Args:
            start_end_epochs (dict): dictionary of epochs
            list_of_files (list): list of files
            parent_dir(ShareDirectoryClient): parent_dir object
        """
        __method_name = inspect.currentframe().f_code.co_name
        try:
            it1_start = start_end_epochs.get("it1_start_epochs")
            it2_start = start_end_epochs.get("it2_start_epochs")
            it3_start = start_end_epochs.get("it3_start_epochs")
            it4_start = start_end_epochs.get("it4_start_epochs")
            it1_end = start_end_epochs.get("it1_end_epochs")
            it2_end = start_end_epochs.get("it2_end_epochs")
            it3_end = start_end_epochs.get("it3_end_epochs")
            it4_end = start_end_epochs.get("it4_end_epochs")
            queue_for_execution = [
                it1_end,
                it2_start,
                it2_end,
                it3_start,
                it3_end,
                it4_start,
                it4_end,
                it1_start,
            ]
            end_epoch_counter = [0, 1, 2, 3]
            start_epoch_counter = [1, 2, 3, 0]
            end_epoch_pool = cycle(end_epoch_counter)
            start_epoch_pool = cycle(start_epoch_counter)

            for _ in range(4):
                end_epochs_list = queue_for_execution.pop(0)
                start_epochs_list = queue_for_execution.pop(0)
                number_of_files_to_scan = min(len(end_epochs_list), len(start_epochs_list))
                end_counter = next(end_epoch_pool)
                start_counter = next(start_epoch_pool)
                for index in range(number_of_files_to_scan):
                    state_manager_obj_for_end_epoch = StateManager(
                        consts.CONNECTION_STRING,
                        "{}_end_epoch_{}".format(self.iterators_name[end_counter], end_epochs_list[index]),
                        self.share_name,
                    )
                    epoch_for_end_file = state_manager_obj_for_end_epoch.get(consts.NETSKOPE_REMOVE_DUPLICATES)
                    state_manager_obj_for_start_epoch = StateManager(
                        consts.CONNECTION_STRING,
                        "{}_start_epoch_{}".format(self.iterators_name[start_counter], start_epochs_list[index]),
                        self.share_name,
                    )
                    epoch_for_start_file = state_manager_obj_for_start_epoch.get(consts.NETSKOPE_REMOVE_DUPLICATES)
                    if epoch_for_end_file is None or epoch_for_start_file is None:
                        applogger.error(
                            "{}(method={}) : {} : Epoch File Returned None."
                            "End Epoch: {} and Start Epoch: {}".format(
                                consts.LOGS_STARTS_WITH,
                                __method_name,
                                consts.NETSKOPE_REMOVE_DUPLICATES,
                                epoch_for_end_file,
                                epoch_for_start_file,
                            )
                        )
                    end_epoch_files = self.filter_files(
                        self.iterators_name[end_counter],
                        epoch_for_end_file,
                        list_of_files,
                    )
                    start_epoch_files = self.filter_files(
                        self.iterators_name[start_counter],
                        epoch_for_start_file,
                        list_of_files,
                    )
                    if len(end_epoch_files) > 0 and len(start_epoch_files) > 0:
                        self.remove_duplicates(end_epoch_files, start_epoch_files, parent_dir)
                    state_manager_obj_for_sentinel_ingestion = StateManager(
                        consts.CONNECTION_STRING,
                        "{}_sentinel_ingestion_epoch".format(self.iterators_name[end_counter]),
                        self.share_name,
                    )
                    state_manager_obj_for_sentinel_ingestion.post(epoch_for_end_file)
                    self.delete_file_from_file_share(
                        "{}_end_epoch_{}".format(self.iterators_name[end_counter], end_epochs_list[index]),
                        parent_dir,
                    )
                    self.delete_file_from_file_share(
                        "{}_start_epoch_{}".format(self.iterators_name[start_counter], start_epochs_list[index]),
                        parent_dir,
                    )

        except NetskopeException:
            applogger.error(
                "{}(method={}) : {} : Error while parsing files and finding duplicates.".format(
                    consts.LOGS_STARTS_WITH,
                    __method_name,
                    consts.NETSKOPE_REMOVE_DUPLICATES,
                )
            )
            raise NetskopeException()
        except Exception as error:
            applogger.error(
                "{}(method={}) : {} : Error while parsing files and finding duplicates, Error-{}.".format(
                    consts.LOGS_STARTS_WITH,
                    __method_name,
                    consts.NETSKOPE_REMOVE_DUPLICATES,
                    error,
                )
            )
            raise NetskopeException()
Fetch the start and end epoch files data, get the file names with potential duplicates and remove duplicates.

        Args:
            start_end_epochs (dict): dictionary of epochs
            list_of_files (list): list of files
            parent_dir(ShareDirectoryClient): parent_dir object
        
Solutions/Netskopev2/Data Connectors/NetskopeDataConnector/StorageToSentinel/remove_duplicates_in_azure_storage.py