async def remove_duplicates_generated_due_to_data_saving_failures(self, index, data, epoch):
        """Remove duplicate data if any due to data saving failure in the previous invocation.

        Args:
            index (str): The iterator name.
            data (dict): The data to check duplicate for.
            epoch (int): The epoch value in the data.

        Raises:
            NetskopeException: Custom Netskope Exception.

        Returns:
            bool: True if data is duplicate else False.
        """
        __method_name = inspect.currentframe().f_code.co_name
        try:
            applogger.info("Checking for Duplicates")
            from azure.storage.fileshare import ShareDirectoryClient

            parent_dir = ShareDirectoryClient.from_connection_string(
                conn_str=consts.CONNECTION_STRING,
                share_name=self.share_name,
                directory_path="",
            )
            list_of_files_response = parent_dir.list_directories_and_files(name_starts_with=index)
            list_of_files = [file["name"] for file in list_of_files_response]
            file_name_with_provided_epoch = None
            epoch_of_file = 0
            for file in list_of_files:
                if (
                    "epoch" not in file
                    and "failed" not in file
                    and int(file.split("_")[-2]) == epoch
                    and int(file.split("_")[-1]) > epoch_of_file
                ):
                    file_name_with_provided_epoch = file
                    epoch_of_file = int(file.split("_")[-1])
            if file_name_with_provided_epoch:
                try:
                    state_manager_obj = StateManager(
                        consts.CONNECTION_STRING, file_name_with_provided_epoch, self.share_name
                    )
                    # Here we are fetching the previously saved data and comparing it with the data
                    # recieved in the current iteration and check if the data is duplicate or not.
                    duplicate_data = state_manager_obj.get(consts.NETSKOPE_TO_AZURE_STORAGE)
                    duplicate_json_data = json.loads(duplicate_data)
                    if duplicate_json_data == data or self.is_response_empty(duplicate_json_data):
                        applogger.error(
                            "{}(method={}) : {} ({}) : The data with epoch-{} and iterator-{} is duplicate.".format(
                                consts.LOGS_STARTS_WITH,
                                __method_name,
                                consts.NETSKOPE_TO_AZURE_STORAGE,
                                self.nskp_data_type_for_logging,
                                epoch,
                                index,
                            )
                        )
                        return True
                    return False
                except json.JSONDecodeError:
                    parent_dir.delete_file(file_name_with_provided_epoch)
                    return False
            return False
        except Exception as error:
            applogger.error(
                "{}(method={}) : {} ({}) : Unknown Error, Error-{}.".format(
                    consts.LOGS_STARTS_WITH,
                    __method_name,
                    consts.NETSKOPE_TO_AZURE_STORAGE,
                    self.nskp_data_type_for_logging,
                    error,
                )
            )
            raise NetskopeException()
Remove duplicate data if any due to data saving failure in the previous invocation.

        Args:
            index (str): The iterator name.
            data (dict): The data to check duplicate for.
            epoch (int): The epoch value in the data.

        Raises:
            NetskopeException: Custom Netskope Exception.

        Returns:
            bool: True if data is duplicate else False.
        
Solutions/Netskopev2/Data Connectors/NetskopeDataConnector/NetskopeToAzureStorage/netskope_to_azure_storage.py