def remove_duplicates(self, list_of_old_files, new_files_to_compare, parent_dir):
        """Remove duplicates from given files.

        Args:
            list_of_old_files (list): list of old files
            new_files_to_compare (str): new file to compare
            parent_dir(ShareDirectoryClient): parent_dir object
        """
        __method_name = inspect.currentframe().f_code.co_name
        try:
            self.delete_duplicate_files(list_of_old_files, parent_dir)
            self.delete_duplicate_files(new_files_to_compare, parent_dir)
            file: str
            for file in list_of_old_files:
                if file.endswith("empty_file"):
                    applogger.error(
                        "{}(method={}) : {} : Empty data is found in file name-{}.".format(
                            consts.LOGS_STARTS_WITH, __method_name, consts.NETSKOPE_REMOVE_DUPLICATES, file
                        )
                    )
                    self.move_to_data_folder(None, file, parent_dir, True)
                    continue
                state_manager_obj_for_old_file = StateManager(consts.CONNECTION_STRING, file, self.share_name)
                old_data_raw = state_manager_obj_for_old_file.get(consts.NETSKOPE_REMOVE_DUPLICATES)
                old_data = json.loads(old_data_raw)
                state_manager_obj_for_new_file = StateManager(
                    consts.CONNECTION_STRING, new_files_to_compare[0], self.share_name
                )
                new_data_raw = state_manager_obj_for_new_file.get(consts.NETSKOPE_REMOVE_DUPLICATES)
                new_data = json.loads(new_data_raw)
                if self.is_response_empty(old_data) or self.is_response_empty(new_data):
                    applogger.error(
                        "{}(method={}) : {} : Empty data is found in a file.".format(
                            consts.LOGS_STARTS_WITH,
                            __method_name,
                            consts.NETSKOPE_REMOVE_DUPLICATES,
                        )
                    )
                    self.move_to_data_folder(old_data, file, parent_dir)
                    continue
                old_list_id = []
                for cur_data in old_data.get("result"):
                    old_list_id.append(cur_data["_id"])
                final_list = []
                for cur_data in new_data.get("result"):
                    if cur_data["_id"] in old_list_id:
                        self.duplicate_count += 1
                        continue
                    final_list.append(cur_data)
                new_data["result"] = final_list
                self.move_to_data_folder(old_data, file, parent_dir)
                state_manager_obj_for_new_file.post(json.dumps(new_data))
            # Fetch the updated new file data.
            if new_files_to_compare[0].endswith("empty_file"):
                self.move_to_data_folder(None, new_files_to_compare[0], parent_dir, True)
                return
            state_manager_obj_for_new_file = StateManager(
                consts.CONNECTION_STRING, new_files_to_compare[0], self.share_name
            )
            new_data_raw = state_manager_obj_for_new_file.get(consts.NETSKOPE_REMOVE_DUPLICATES)
            new_data = json.loads(new_data_raw)
            # move the new file to data folder.
            self.move_to_data_folder(new_data, new_files_to_compare[0], parent_dir)

        except Exception as error:
            applogger.error(
                "{}(method={}) : {} : Error while removing duplicates from azure storage, Error-{}.".format(
                    consts.LOGS_STARTS_WITH,
                    __method_name,
                    consts.NETSKOPE_REMOVE_DUPLICATES,
                    error,
                )
            )
            raise NetskopeException()
Remove duplicates from given files.

        Args:
            list_of_old_files (list): list of old files
            new_files_to_compare (str): new file to compare
            parent_dir(ShareDirectoryClient): parent_dir object
        
Solutions/Netskopev2/Data Connectors/NetskopeDataConnector/StorageToSentinel/remove_duplicates_in_azure_storage.py