def list_file_names_and_remove_duplicate_data(self):
        """Code for removing duplicates from azure storage."""
        __method_name = inspect.currentframe().f_code.co_name
        try:
            parent_dir = ShareDirectoryClient.from_connection_string(
                conn_str=consts.CONNECTION_STRING,
                share_name=self.share_name,
                directory_path="",
            )
            all_file_list = self.list_file_names_in_file_share(parent_dir)
            list_of_epoch_files = []
            for file_name in all_file_list:
                if "_start_epoch_" in file_name or "_end_epoch_" in file_name:
                    list_of_epoch_files.append(file_name)
            applogger.info("The list of epoch files are {}".format(list_of_epoch_files))
            if len(list_of_epoch_files) > 0:
                dict_of_iter_epochs = self.return_list_of_iterator_files(list_of_epoch_files)
                self.parse_files_and_find_duplicates(dict_of_iter_epochs, all_file_list, parent_dir)
            applogger.info(
                "{}(method={}) : {} : Removed duplicate counts are {}.".format(
                    consts.LOGS_STARTS_WITH,
                    __method_name,
                    consts.NETSKOPE_REMOVE_DUPLICATES,
                    self.duplicate_count,
                )
            )
        except NetskopeException:
            applogger.error(
                "{}(method={}) : {} : Failed to remove duplicates from azure storage.".format(
                    consts.LOGS_STARTS_WITH,
                    __method_name,
                    consts.NETSKOPE_REMOVE_DUPLICATES,
                )
            )
            raise NetskopeException()
        except Exception as error:
            applogger.error(
                "{}(method={}) : {} : Failed to remove duplicates from azure storage, Error-{}.".format(
                    consts.LOGS_STARTS_WITH,
                    __method_name,
                    consts.NETSKOPE_REMOVE_DUPLICATES,
                    error,
                )
            )
            raise NetskopeException()
Code for removing duplicates from azure storage.
Solutions/Netskopev2/Data Connectors/NetskopeDataConnector/StorageToSentinel/remove_duplicates_in_azure_storage.py