/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.ambari.server.security.ldap;

import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.naming.NamingException;
import javax.naming.directory.Attributes;
import javax.naming.directory.SearchControls;

import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.configuration.LdapUsernameCollisionHandlingBehavior;
import org.apache.ambari.server.ldap.domain.AmbariLdapConfiguration;
import org.apache.ambari.server.security.authorization.AmbariLdapUtils;
import org.apache.ambari.server.security.authorization.Group;
import org.apache.ambari.server.security.authorization.LdapServerProperties;
import org.apache.ambari.server.security.authorization.User;
import org.apache.ambari.server.security.authorization.Users;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.ldap.control.PagedResultsDirContextProcessor;
import org.springframework.ldap.core.AttributesMapper;
import org.springframework.ldap.core.ContextMapper;
import org.springframework.ldap.core.DirContextAdapter;
import org.springframework.ldap.core.LdapTemplate;
import org.springframework.ldap.core.support.LdapContextSource;
import org.springframework.ldap.filter.AndFilter;
import org.springframework.ldap.filter.EqualsFilter;
import org.springframework.ldap.filter.Filter;
import org.springframework.ldap.filter.HardcodedFilter;
import org.springframework.ldap.filter.LikeFilter;
import org.springframework.ldap.filter.OrFilter;
import org.springframework.ldap.support.LdapUtils;
import org.springframework.security.core.userdetails.UsernameNotFoundException;

import com.google.common.collect.Sets;
import com.google.inject.Inject;
import com.google.inject.Provider;

/**
 * Provides users, groups and membership population from LDAP catalog.
 */
public class AmbariLdapDataPopulator {
  /**
   * Log.
   */
  private static final Logger LOG = LoggerFactory.getLogger(AmbariLdapDataPopulator.class);

  /**
   * Ambari configuration.
   */
  private Provider<AmbariLdapConfiguration> configurationProvider;

  /**
   * Highlevel facade for management of users and groups.
   */
  private Users users;

  /**
   * LDAP specific properties.
   */
  protected LdapServerProperties ldapServerProperties;

  /**
   * LDAP template for making search queries.
   */
  private LdapTemplate ldapTemplate;

  // Constants
  private static final String UID_ATTRIBUTE = "uid";
  private static final String OBJECT_CLASS_ATTRIBUTE = "objectClass";
  private static final int USERS_PAGE_SIZE = 500;
  private static final String SYSTEM_PROPERTY_DISABLE_ENDPOINT_IDENTIFICATION = "com.sun.jndi.ldap.object.disableEndpointIdentification";

  // REGEXP to check member attribute starts with "cn=" or "uid=" - case insensitive
  private static final String IS_MEMBER_DN_REGEXP = "^(?i)(uid|cn|%s|%s)=.*$";

  private static final String MEMBER_ATTRIBUTE_REPLACE_STRING = "${member}";
  private static final String MEMBER_ATTRIBUTE_VALUE_PLACEHOLDER = "{member}";

  /**
   * Construct an AmbariLdapDataPopulator.
   *
   * @param configurationProvider the Ambari configuration
   * @param users                 utility that provides access to Users
   */
  @Inject
  public AmbariLdapDataPopulator(Provider<AmbariLdapConfiguration> configurationProvider, Users users) {
    this.configurationProvider = configurationProvider;
    this.users = users;
    this.ldapServerProperties = null;
  }

  /**
   * Load the initial LDAP configuration if the JPA infrastructure is initialized.
   */
  synchronized private LdapServerProperties getLdapProperties() {
    if (ldapServerProperties == null) {
      ldapServerProperties = getConfiguration().getLdapServerProperties();
    }
    return ldapServerProperties;
  }

  /**
   * Check if LDAP is enabled in server properties.
   *
   * @return true if enabled
   */
  public boolean isLdapEnabled() {
    if (!getConfiguration().ldapEnabled()) {
      return false;
    }
    try {
      final LdapTemplate ldapTemplate = loadLdapTemplate();
      ldapTemplate.search(getLdapProperties().getBaseDN(), "uid=dummy_search", new AttributesMapper() {

        @Override
        public Object mapFromAttributes(Attributes arg0) throws NamingException {
          return null;
        }
      });
      return true;
    } catch (Exception ex) {
      LOG.error("Could not connect to LDAP server - " + ex.getMessage());
      return false;
    }
  }

  /**
   * Retrieves information about external groups and users and their synced/unsynced state.
   *
   * @return dto with information
   */
  public LdapSyncDto getLdapSyncInfo() {
    final LdapSyncDto syncInfo = new LdapSyncDto();

    final Map<String, Group> internalGroupsMap = getInternalGroups();
    final Set<LdapGroupDto> externalGroups = getExternalLdapGroupInfo();
    for (LdapGroupDto externalGroup : externalGroups) {
      if (internalGroupsMap.containsKey(externalGroup.getGroupName())
          && internalGroupsMap.get(externalGroup.getGroupName()).isLdapGroup()) {
        externalGroup.setSynced(true);
      } else {
        externalGroup.setSynced(false);
      }
    }

    final Map<String, User> internalUsersMap = getInternalUsers();
    final Set<LdapUserDto> externalUsers = getExternalLdapUserInfo();
    for (LdapUserDto externalUser : externalUsers) {
      String userName = externalUser.getUserName();
      if (internalUsersMap.containsKey(userName)
          && internalUsersMap.get(userName).isLdapUser()) {
        externalUser.setSynced(true);
      } else {
        externalUser.setSynced(false);
      }
    }

    syncInfo.setGroups(externalGroups);
    syncInfo.setUsers(externalUsers);
    return syncInfo;
  }

  /**
   * Performs synchronization of all groups.
   *
   * @param collectIgnoredUsers true, to collect the set of existing users that would normally be ignored;
   *                            false, to continue to ignore them
   * @throws AmbariException if synchronization failed for any reason
   */
  public LdapBatchDto synchronizeAllLdapGroups(LdapBatchDto batchInfo, boolean collectIgnoredUsers) throws AmbariException {
    LOG.trace("Synchronize All LDAP groups...");
    Set<LdapGroupDto> externalLdapGroupInfo = getExternalLdapGroupInfo();

    final Map<String, Group> internalGroupsMap = getInternalGroups();
    final Map<String, User> internalUsersMap = getInternalUsers();

    for (LdapGroupDto groupDto : externalLdapGroupInfo) {
      addLdapGroup(batchInfo, internalGroupsMap, groupDto);
      refreshGroupMembers(batchInfo, groupDto, internalUsersMap, internalGroupsMap, null, false, collectIgnoredUsers);
    }
    for (Entry<String, Group> internalGroup : internalGroupsMap.entrySet()) {
      if (internalGroup.getValue().isLdapGroup()) {
        LdapGroupDto groupDto = new LdapGroupDto();
        groupDto.setGroupName(internalGroup.getValue().getGroupName());
        batchInfo.getGroupsToBeRemoved().add(groupDto);
      }
    }

    return batchInfo;
  }

  /**
   * Performs synchronization of given sets of all users.
   *
   * @param collectIgnoredUsers true, to collect the set of existing users that would normally be ignored;
   *                            false, to continue to ignore them
   * @throws AmbariException if synchronization failed for any reason
   */
  public LdapBatchDto synchronizeAllLdapUsers(LdapBatchDto batchInfo, boolean collectIgnoredUsers) throws AmbariException {
    LOG.trace("Synchronize All LDAP users...");
    Set<LdapUserDto> externalLdapUserInfo = getExternalLdapUserInfo();
    Map<String, User> internalUsersMap = getInternalUsers();

    for (LdapUserDto userDto : externalLdapUserInfo) {
      String userName = userDto.getUserName();
      if (internalUsersMap.containsKey(userName)) {
        final User user = internalUsersMap.get(userName);
        if (user != null && !user.isLdapUser()) {
          if (LdapUsernameCollisionHandlingBehavior.SKIP == getConfiguration().syncCollisionHandlingBehavior()) {
            LOG.info("User '{}' skipped because it is local user", userName);
            batchInfo.getUsersSkipped().add(userDto);
          } else {
            batchInfo.getUsersToBecomeLdap().add(userDto);
            LOG.trace("Convert user '{}' to LDAP user.", userName);
          }
        } else if (collectIgnoredUsers) {
          batchInfo.getUsersIgnored().add(userDto);
        }
        internalUsersMap.remove(userName);
      } else {
        batchInfo.getUsersToBeCreated().add(userDto);
      }
    }
    for (Entry<String, User> internalUser : internalUsersMap.entrySet()) {
      if (internalUser.getValue().isLdapUser()) {
        LdapUserDto userDto = new LdapUserDto();
        userDto.setUserName(internalUser.getValue().getUserName());
        userDto.setDn(null);  // Setting to null since we do not know what the DN for this user was.
        batchInfo.getUsersToBeRemoved().add(userDto);
      }
    }

    return batchInfo;
  }

  /**
   * Performs synchronization of given set of groupnames.
   *
   * @param groups              set of groups to synchronize
   * @param collectIgnoredUsers true, to collect the set of existing users that would normally be ignored;
   *                            false, to continue to ignore them
   * @throws AmbariException if synchronization failed for any reason
   */
  public LdapBatchDto synchronizeLdapGroups(Set<String> groups, LdapBatchDto batchInfo, boolean collectIgnoredUsers) throws AmbariException {
    LOG.trace("Synchronize LDAP groups...");
    final Set<LdapGroupDto> specifiedGroups = new HashSet<>();
    for (String group : groups) {
      Set<LdapGroupDto> groupDtos = getLdapGroups(group);
      if (groupDtos.isEmpty()) {
        throw new AmbariException("Couldn't sync LDAP group " + group
            + ", it doesn't exist");
      }
      specifiedGroups.addAll(groupDtos);
    }

    final Map<String, Group> internalGroupsMap = getInternalGroups();
    final Map<String, User> internalUsersMap = getInternalUsers();

    for (LdapGroupDto groupDto : specifiedGroups) {
      addLdapGroup(batchInfo, internalGroupsMap, groupDto);
      refreshGroupMembers(batchInfo, groupDto, internalUsersMap, internalGroupsMap, null, true, collectIgnoredUsers);
    }

    return batchInfo;
  }

  /**
   * Performs synchronization of given set of user names.
   *
   * @param users               set of users to synchronize
   * @param collectIgnoredUsers true, to collect the set of existing users that would normally be ignored;
   *                            false, to continue to ignore them
   * @throws AmbariException if synchronization failed for any reason
   */
  public LdapBatchDto synchronizeLdapUsers(Set<String> users, LdapBatchDto batchInfo, boolean collectIgnoredUsers) throws AmbariException {
    LOG.trace("Synchronize LDAP users...");
    final Set<LdapUserDto> specifiedUsers = new HashSet<>();

    for (String user : users) {
      Set<LdapUserDto> userDtos = getLdapUsers(user);
      if (userDtos.isEmpty()) {
        throw new AmbariException("Couldn't sync LDAP user " + user
            + ", it doesn't exist");
      }
      specifiedUsers.addAll(userDtos);
    }

    final Map<String, User> internalUsersMap = getInternalUsers();
    for (LdapUserDto userDto : specifiedUsers) {
      String userName = userDto.getUserName();
      if (internalUsersMap.containsKey(userName)) {
        final User user = internalUsersMap.get(userName);
        if (user != null && !user.isLdapUser()) {
          if (LdapUsernameCollisionHandlingBehavior.SKIP == getConfiguration().syncCollisionHandlingBehavior()) {
            LOG.info("User '{}' skipped because it is local user", userName);
            batchInfo.getUsersSkipped().add(userDto);
          } else {
            batchInfo.getUsersToBecomeLdap().add(userDto);
          }
        } else if (collectIgnoredUsers) {
          batchInfo.getUsersIgnored().add(userDto);
        }
        internalUsersMap.remove(userName);
      } else {
        batchInfo.getUsersToBeCreated().add(userDto);
      }
    }

    return batchInfo;
  }

  /**
   * Performs synchronization of existent users and groups.
   *
   * @param collectIgnoredUsers true, to collect the set of existing users that would normally be ignored;
   *                            false, to continue to ignore them
   * @throws AmbariException if synchronization failed for any reason
   */
  public LdapBatchDto synchronizeExistingLdapGroups(LdapBatchDto batchInfo, boolean collectIgnoredUsers) throws AmbariException {
    LOG.trace("Synchronize Existing LDAP groups...");
    final Map<String, Group> internalGroupsMap = getInternalGroups();
    final Map<String, User> internalUsersMap = getInternalUsers();

    final Set<Group> internalGroupSet = Sets.newHashSet(internalGroupsMap.values());

    for (Group group : internalGroupSet) {
      if (group.isLdapGroup()) {
        Set<LdapGroupDto> groupDtos = getLdapGroups(group.getGroupName());
        if (groupDtos.isEmpty()) {
          LdapGroupDto groupDto = new LdapGroupDto();
          groupDto.setGroupName(group.getGroupName());
          batchInfo.getGroupsToBeRemoved().add(groupDto);
        } else {
          LdapGroupDto groupDto = groupDtos.iterator().next();
          refreshGroupMembers(batchInfo, groupDto, internalUsersMap, internalGroupsMap, null, true, collectIgnoredUsers);
        }
      }
    }

    return batchInfo;
  }

  /**
   * Performs synchronization of existent users and groups.
   *
   * @param collectIgnoredUsers true, to collect the set of existing users that would normally be ignored;
   *                            false, to continue to ignore them
   * @throws AmbariException if synchronization failed for any reason
   */
  public LdapBatchDto synchronizeExistingLdapUsers(LdapBatchDto batchInfo, boolean collectIgnoredUsers) throws AmbariException {
    LOG.trace("Synchronize Existing LDAP users...");
    final Map<String, User> internalUsersMap = getInternalUsers();

    for (User user : internalUsersMap.values()) {
      if (user.isLdapUser()) {
        Set<LdapUserDto> userDtos = getLdapUsers(user.getUserName());
        if (userDtos.isEmpty()) {
          LdapUserDto userDto = new LdapUserDto();
          userDto.setUserName(user.getUserName());
          userDto.setDn(null);  // Setting to null since we do not know what the DN for this user was.
          batchInfo.getUsersToBeRemoved().add(userDto);
        } else if (collectIgnoredUsers) {
          LdapUserDto userDto = new LdapUserDto();
          userDto.setUserName(user.getUserName());
          batchInfo.getUsersIgnored().add(userDto);
        }
      }
    }

    return batchInfo;
  }

  /**
   * Check group members of the synced group: add missing ones and remove the ones absent in external LDAP.
   *
   * @param batchInfo             batch update object
   * @param group                 ldap group
   * @param internalUsers         map of internal users
   * @param groupMemberAttributes set of group member attributes that have already been refreshed
   * @param recursive             if disabled, it won't refresh members recursively (its not needed in case of all groups are processed)
   * @param collectIgnoredUsers   true, to collect the set of existing users that would normally be ignored;
   *                              false, to continue to ignore them
   * @throws AmbariException if group refresh failed
   */
  protected void refreshGroupMembers(LdapBatchDto batchInfo, LdapGroupDto group, Map<String, User> internalUsers,
                                     Map<String, Group> internalGroupsMap, Set<String> groupMemberAttributes, boolean recursive,
                                     boolean collectIgnoredUsers)
      throws AmbariException {
    Set<LdapUserDto> externalMembers = new HashSet<>();

    if (groupMemberAttributes == null) {
      groupMemberAttributes = new HashSet<>();
    }

    for (String memberAttributeValue : group.getMemberAttributes()) {
      LdapUserDto groupMember = getLdapUserByMemberAttr(memberAttributeValue);
      if (groupMember != null) {
        externalMembers.add(groupMember);
      } else {
        // if we haven't already processed this group
        if (recursive && !groupMemberAttributes.contains(memberAttributeValue)) {
          // if the member is another group then add all of its members
          LdapGroupDto subGroup = getLdapGroupByMemberAttr(memberAttributeValue);
          if (subGroup != null) {
            groupMemberAttributes.add(memberAttributeValue);
            addLdapGroup(batchInfo, internalGroupsMap, subGroup);
            refreshGroupMembers(batchInfo, subGroup, internalUsers, internalGroupsMap, groupMemberAttributes, true, collectIgnoredUsers);
          }
        }
      }
    }
    String groupName = group.getGroupName();
    final Map<String, User> internalMembers = getInternalMembers(groupName);
    for (LdapUserDto externalMember : externalMembers) {
      String userName = externalMember.getUserName();
      if (internalUsers.containsKey(userName)) {
        final User user = internalUsers.get(userName);
        if (user == null) {
          // user is fresh and is already added to batch info
          if (!internalMembers.containsKey(userName)) {
            batchInfo.getMembershipToAdd().add(new LdapUserGroupMemberDto(groupName, externalMember.getUserName()));
          }
          continue;
        }
        if (!user.isLdapUser()) {
          if (LdapUsernameCollisionHandlingBehavior.SKIP == getConfiguration().syncCollisionHandlingBehavior()) {
            // existing user can not be converted to ldap user, so skip it
            LOG.info("User '{}' skipped because it is local user", userName);
            batchInfo.getUsersSkipped().add(externalMember);
            continue; // and remove from group
          } else {
            batchInfo.getUsersToBecomeLdap().add(externalMember);
          }
        } else if (collectIgnoredUsers) {
          batchInfo.getUsersIgnored().add(externalMember);
        }
        if (!internalMembers.containsKey(userName)) {
          batchInfo.getMembershipToAdd().add(new LdapUserGroupMemberDto(groupName, externalMember.getUserName()));
        }
        internalMembers.remove(userName);
      } else {
        batchInfo.getUsersToBeCreated().add(externalMember);
        batchInfo.getMembershipToAdd().add(new LdapUserGroupMemberDto(groupName, externalMember.getUserName()));
      }
    }
    for (Entry<String, User> userToBeUnsynced : internalMembers.entrySet()) {
      final User user = userToBeUnsynced.getValue();
      batchInfo.getMembershipToRemove().add(new LdapUserGroupMemberDto(groupName, user.getUserName()));
    }
  }

  /**
   * Get the set of LDAP groups for the given group name.
   *
   * @param groupName the group name
   * @return the set of LDAP groups for the given name
   */
  protected Set<LdapGroupDto> getLdapGroups(String groupName) {
    LdapServerProperties ldapServerProperties = getLdapProperties();
    Filter groupObjectFilter = new EqualsFilter(OBJECT_CLASS_ATTRIBUTE,
        ldapServerProperties.getGroupObjectClass());
    Filter groupNameFilter = new LikeFilter(ldapServerProperties.getGroupNamingAttr(), groupName);
    return getFilteredLdapGroups(ldapServerProperties.getBaseDN(), groupObjectFilter, groupNameFilter);
  }

  /**
   * Get the set of LDAP users for the given user name.
   *
   * @param username the user name
   * @return the set of LDAP users for the given name
   */
  protected Set<LdapUserDto> getLdapUsers(String username) {
    LdapServerProperties ldapServerProperties = getLdapProperties();
    Filter userObjectFilter = new EqualsFilter(OBJECT_CLASS_ATTRIBUTE, ldapServerProperties.getUserObjectClass());
    Filter userNameFilter = new LikeFilter(ldapServerProperties.getUsernameAttribute(), username);
    return getFilteredLdapUsers(ldapServerProperties.getBaseDN(), userObjectFilter, userNameFilter);
  }

  /**
   * Get the LDAP user member for the given member attribute.
   *
   * @param memberAttributeValue the member attribute value
   * @return the user for the given member attribute; null if not found
   */
  protected LdapUserDto getLdapUserByMemberAttr(String memberAttributeValue) {
    LdapServerProperties ldapServerProperties = getLdapProperties();
    Set<LdapUserDto> filteredLdapUsers;

    memberAttributeValue = getUniqueIdByMemberPattern(memberAttributeValue,
        ldapServerProperties.getSyncUserMemberReplacePattern());
    Filter syncMemberFilter = createCustomMemberFilter(memberAttributeValue,
        ldapServerProperties.getSyncUserMemberFilter());

    if (memberAttributeValue != null && syncMemberFilter != null) {
      LOG.trace("Use custom filter '{}' for getting member user with default baseDN ('{}')",
          syncMemberFilter.encode(), ldapServerProperties.getBaseDN());
      filteredLdapUsers = getFilteredLdapUsers(ldapServerProperties.getBaseDN(), syncMemberFilter);
    } else if (memberAttributeValue != null && isMemberAttributeBaseDn(memberAttributeValue)) {
      LOG.trace("Member can be used as baseDn: {}", memberAttributeValue);
      Filter filter = new EqualsFilter(OBJECT_CLASS_ATTRIBUTE, ldapServerProperties.getUserObjectClass());
      filteredLdapUsers = getFilteredLdapUsers(memberAttributeValue, filter);
    } else {
      LOG.trace("Member cannot be used as baseDn: {}", memberAttributeValue);
      Filter filter = new AndFilter()
          .and(new EqualsFilter(OBJECT_CLASS_ATTRIBUTE, ldapServerProperties.getUserObjectClass()))
          .and(new EqualsFilter(ldapServerProperties.getUsernameAttribute(), memberAttributeValue));
      filteredLdapUsers = getFilteredLdapUsers(ldapServerProperties.getBaseDN(), filter);
    }
    return (filteredLdapUsers.isEmpty()) ? null : filteredLdapUsers.iterator().next();
  }

  /**
   * Get the LDAP group member for the given member attribute.
   *
   * @param memberAttributeValue the member attribute value
   * @return the group for the given member attribute; null if not found
   */
  protected LdapGroupDto getLdapGroupByMemberAttr(String memberAttributeValue) {
    LdapServerProperties ldapServerProperties = getLdapProperties();
    Set<LdapGroupDto> filteredLdapGroups;

    memberAttributeValue = getUniqueIdByMemberPattern(memberAttributeValue,
        ldapServerProperties.getSyncGroupMemberReplacePattern());
    Filter syncMemberFilter = createCustomMemberFilter(memberAttributeValue,
        ldapServerProperties.getSyncGroupMemberFilter());

    if (memberAttributeValue != null && syncMemberFilter != null) {
      LOG.trace("Use custom filter '{}' for getting member group with default baseDN ('{}')",
          syncMemberFilter.encode(), ldapServerProperties.getBaseDN());
      filteredLdapGroups = getFilteredLdapGroups(ldapServerProperties.getBaseDN(), syncMemberFilter);
    } else if (memberAttributeValue != null && isMemberAttributeBaseDn(memberAttributeValue)) {
      LOG.trace("Member can be used as baseDn: {}", memberAttributeValue);
      Filter filter = new EqualsFilter(OBJECT_CLASS_ATTRIBUTE, ldapServerProperties.getGroupObjectClass());
      filteredLdapGroups = getFilteredLdapGroups(memberAttributeValue, filter);
    } else {
      LOG.trace("Member cannot be used as baseDn: {}", memberAttributeValue);
      filteredLdapGroups = getFilteredLdapGroups(ldapServerProperties.getBaseDN(),
          new EqualsFilter(OBJECT_CLASS_ATTRIBUTE, ldapServerProperties.getGroupObjectClass()),
          getMemberFilter(memberAttributeValue));
    }

    return (filteredLdapGroups.isEmpty()) ? null : filteredLdapGroups.iterator().next();
  }

  /**
   * Use custom member filter. Replace {member} with the member attribute.
   * E.g.: (&(objectclass=posixaccount)(dn={member})) -> (&(objectclass=posixaccount)(dn=cn=mycn,dc=apache,dc=org))
   */
  protected Filter createCustomMemberFilter(String memberAttributeValue, String syncMemberFilter) {
    Filter filter = null;
    if (StringUtils.isNotEmpty(syncMemberFilter)) {
      filter = new HardcodedFilter(syncMemberFilter.replace(MEMBER_ATTRIBUTE_VALUE_PLACEHOLDER, memberAttributeValue));
    }
    return filter;
  }

  /**
   * Replace memberAttribute value by a custom pattern to get the DN or id (like memberUid) of a user/group.
   * E.g.: memberAttribute="<sid=...><guid=...>,cn=mycn,dc=org,dc=apache"
   * Apply on (?<sid>.*);(?<guid>.*);(?<member>.*) pattern, then the result will be: "${member}"
   */
  protected String getUniqueIdByMemberPattern(String memberAttributeValue, String pattern) {
    if (StringUtils.isNotEmpty(memberAttributeValue) && StringUtils.isNotEmpty(pattern)) {
      try {
        Pattern p = Pattern.compile(pattern);
        Matcher m = p.matcher(memberAttributeValue);
        LOG.debug("Apply replace pattern '{}' on '{}' membership attribbute value.", memberAttributeValue, pattern);
        if (m.matches()) {
          memberAttributeValue = m.replaceAll(MEMBER_ATTRIBUTE_REPLACE_STRING);
          LOG.debug("Membership attribute value after replace pattern applied: '{}'", memberAttributeValue);
        } else {
          LOG.warn("Membership attribute value pattern is not matched ({}) on '{}'", pattern, memberAttributeValue);
        }
      } catch (Exception e) {
        LOG.error("Error during replace memberAttribute '{}' with pattern '{}'", memberAttributeValue, pattern);
      }
    }
    return memberAttributeValue;
  }

  /**
   * Removes synced users which are not present in any of group.
   *
   * @throws AmbariException
   */
  protected void cleanUpLdapUsersWithoutGroup() throws AmbariException {
    final List<User> allUsers = users.getAllUsers();
    for (User user : allUsers) {
      if (user.isLdapUser() && user.getGroups().isEmpty()) {
        users.removeUser(user);
      }
    }
  }

  // Utility methods

  protected void addLdapGroup(LdapBatchDto batchInfo, Map<String, Group> internalGroupsMap, LdapGroupDto groupDto) {
    String groupName = groupDto.getGroupName();
    if (internalGroupsMap.containsKey(groupName)) {
      final Group group = internalGroupsMap.get(groupName);
      if (!group.isLdapGroup()) {
        batchInfo.getGroupsToBecomeLdap().add(groupDto);
        LOG.trace("Convert group '{}' to LDAP group.", groupName);
      }
      internalGroupsMap.remove(groupName);
      batchInfo.getGroupsProcessedInternal().add(groupDto);
    } else {
      if (!batchInfo.getGroupsProcessedInternal().contains(groupDto)) {
        batchInfo.getGroupsToBeCreated().add(groupDto);
      }
    }
  }

  /**
   * Determines that the member attribute can be used as a 'dn'
   */
  protected boolean isMemberAttributeBaseDn(String memberAttributeValue) {
    LdapServerProperties ldapServerProperties = getLdapProperties();
    Pattern pattern = Pattern.compile(String.format(IS_MEMBER_DN_REGEXP,
        ldapServerProperties.getUsernameAttribute(), ldapServerProperties.getGroupNamingAttr()));
    return pattern.matcher(memberAttributeValue).find();
  }

  /**
   * Retrieves groups from external LDAP server.
   *
   * @return set of info about LDAP groups
   */
  protected Set<LdapGroupDto> getExternalLdapGroupInfo() {
    LdapServerProperties ldapServerProperties = getLdapProperties();
    EqualsFilter groupObjectFilter = new EqualsFilter(OBJECT_CLASS_ATTRIBUTE,
        ldapServerProperties.getGroupObjectClass());
    return getFilteredLdapGroups(ldapServerProperties.getBaseDN(), groupObjectFilter);
  }

  // get a filter based on the given member attribute
  private Filter getMemberFilter(String memberAttributeValue) {
    LdapServerProperties ldapServerProperties = getLdapProperties();
    String dnAttribute = ldapServerProperties.getDnAttribute();

    return new OrFilter().or(new EqualsFilter(dnAttribute, memberAttributeValue)).
        or(new EqualsFilter(UID_ATTRIBUTE, memberAttributeValue));
  }

  private Set<LdapGroupDto> getFilteredLdapGroups(String baseDn, Filter... filters) {
    AndFilter andFilter = new AndFilter();
    for (Filter filter : filters) {
      andFilter.and(filter);
    }
    return getFilteredLdapGroups(baseDn, andFilter);
  }

  private Set<LdapGroupDto> getFilteredLdapGroups(String baseDn, Filter filter) {
    final Set<LdapGroupDto> groups = new HashSet<>();
    final LdapTemplate ldapTemplate = loadLdapTemplate();
    LdapServerProperties ldapServerProperties = getLdapProperties();
    LOG.trace("LDAP Group Query - Base DN: '{}' ; Filter: '{}'", baseDn, filter.encode());
    ldapTemplate.search(baseDn, filter.encode(),
        new LdapGroupContextMapper(groups, ldapServerProperties));
    return groups;
  }

  /**
   * Retrieves users from external LDAP server.
   *
   * @return set of info about LDAP users
   */
  protected Set<LdapUserDto> getExternalLdapUserInfo() {
    LdapServerProperties ldapServerProperties = getLdapProperties();
    EqualsFilter userObjectFilter = new EqualsFilter(OBJECT_CLASS_ATTRIBUTE,
        ldapServerProperties.getUserObjectClass());
    return getFilteredLdapUsers(ldapServerProperties.getBaseDN(), userObjectFilter);
  }

  private Set<LdapUserDto> getFilteredLdapUsers(String baseDn, Filter... filters) {
    AndFilter andFilter = new AndFilter();
    for (Filter filter : filters) {
      andFilter.and(filter);
    }
    return getFilteredLdapUsers(baseDn, andFilter);
  }

  private Set<LdapUserDto> getFilteredLdapUsers(String baseDn, Filter filter) {
    final Set<LdapUserDto> users = new HashSet<>();
    final LdapTemplate ldapTemplate = loadLdapTemplate();
    LdapServerProperties ldapServerProperties = getLdapProperties();
    PagedResultsDirContextProcessor processor = createPagingProcessor();
    SearchControls searchControls = new SearchControls();
    searchControls.setReturningObjFlag(true);
    searchControls.setSearchScope(SearchControls.SUBTREE_SCOPE);
    LdapUserContextMapper ldapUserContextMapper = new LdapUserContextMapper(ldapServerProperties);
    String encodedFilter = filter.encode();

    do {
      LOG.trace("LDAP User Query - Base DN: '{}' ; Filter: '{}'", baseDn, encodedFilter);
      List dtos = ldapServerProperties.isPaginationEnabled() ?
          ldapTemplate.search(LdapUtils.newLdapName(baseDn), encodedFilter, searchControls, ldapUserContextMapper, processor) :
          ldapTemplate.search(LdapUtils.newLdapName(baseDn), encodedFilter, searchControls, ldapUserContextMapper);
      for (Object dto : dtos) {
        if (dto != null) {
          users.add((LdapUserDto) dto);
        }
      }
    } while (ldapServerProperties.isPaginationEnabled()
        && (processor.getCookie() != null) && (processor.getCookie().getCookie() != null));
    return users;
  }

  /**
   * Creates a map of internal groups.
   *
   * @return map of GroupName-Group pairs
   */
  protected Map<String, Group> getInternalGroups() {
    final List<Group> internalGroups = users.getAllGroups();
    final Map<String, Group> internalGroupsMap = new HashMap<>();
    for (Group group : internalGroups) {
      internalGroupsMap.put(group.getGroupName(), group);
    }
    return internalGroupsMap;
  }

  /**
   * Creates a map of internal users.
   *
   * @return map of UserName-User pairs
   */
  protected Map<String, User> getInternalUsers() {
    final List<User> internalUsers = users.getAllUsers();
    final Map<String, User> internalUsersMap = new HashMap<>();
    LOG.trace("Get all users from Ambari Server.");
    for (User user : internalUsers) {
      internalUsersMap.put(user.getUserName(), user);
    }
    return internalUsersMap;
  }

  /**
   * Creates a map of internal users present in specified group.
   *
   * @param groupName group name
   * @return map of UserName-User pairs
   */
