/*
 * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
 * agreements. See the NOTICE file distributed with this work for additional information regarding
 * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance with the License. You may obtain a
 * copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */
package org.apache.geode.internal.cache.partitioned.fixed;

import static java.util.Arrays.asList;
import static java.util.Collections.singletonList;
import static org.apache.geode.cache.FixedPartitionAttributes.createFixedPartition;
import static org.apache.geode.cache.RegionShortcut.PARTITION;
import static org.apache.geode.cache.partition.PartitionRegionHelper.getLocalData;
import static org.apache.geode.distributed.ConfigurationProperties.SERIALIZABLE_OBJECT_FILTER;
import static org.apache.geode.test.dunit.IgnoredException.addIgnoredException;
import static org.apache.geode.test.dunit.VM.getVM;
import static org.apache.geode.test.dunit.rules.DistributedRule.getDistributedSystemProperties;
import static org.apache.geode.util.internal.GeodeGlossary.GEMFIRE_PREFIX;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.catchThrowable;
import static org.assertj.core.api.Assertions.fail;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.io.Serializable;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.time.Instant;
import java.time.ZoneId;
import java.util.Date;
import java.util.Locale;
import java.util.Objects;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.TimeoutException;

import org.apache.commons.lang3.StringUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;

import org.apache.geode.DataSerializable;
import org.apache.geode.DataSerializer;
import org.apache.geode.cache.Cache;
import org.apache.geode.cache.CacheFactory;
import org.apache.geode.cache.DuplicatePrimaryPartitionException;
import org.apache.geode.cache.EntryNotFoundException;
import org.apache.geode.cache.EntryOperation;
import org.apache.geode.cache.FixedPartitionAttributes;
import org.apache.geode.cache.FixedPartitionResolver;
import org.apache.geode.cache.PartitionAttributesFactory;
import org.apache.geode.cache.PartitionResolver;
import org.apache.geode.cache.PartitionedRegionStorageException;
import org.apache.geode.cache.Region;
import org.apache.geode.cache.control.RebalanceOperation;
import org.apache.geode.cache.control.ResourceManager;
import org.apache.geode.cache.partition.PartitionNotAvailableException;
import org.apache.geode.internal.cache.PartitionedRegion;
import org.apache.geode.internal.cache.PartitionedRegionDataStore;
import org.apache.geode.internal.cache.partitioned.PartitionedRegionObserverAdapter;
import org.apache.geode.internal.cache.partitioned.PartitionedRegionObserverHolder;
import org.apache.geode.internal.cache.xmlcache.Declarable2;
import org.apache.geode.test.dunit.AsyncInvocation;
import org.apache.geode.test.dunit.VM;
import org.apache.geode.test.dunit.Wait;
import org.apache.geode.test.dunit.rules.DistributedBlackboard;
import org.apache.geode.test.dunit.rules.DistributedDiskDirRule;
import org.apache.geode.test.dunit.rules.DistributedErrorCollector;
import org.apache.geode.test.dunit.rules.DistributedReference;
import org.apache.geode.test.dunit.rules.DistributedRestoreSystemProperties;
import org.apache.geode.test.dunit.rules.DistributedRule;
import org.apache.geode.test.junit.categories.PartitioningTest;

/**
 * This Dunit test class have multiple tests to tests different validations of static partitioning
 */
@Category(PartitioningTest.class)
@SuppressWarnings({"serial", "CodeBlock2Expr",
    "UseOfObsoleteDateTimeApi"})
public class FixedPartitioningDUnitTest implements Serializable {

  private static final String REGION_NAME = "Quarter";

  private static final String QUARTER_1 = "Q1";
  private static final String QUARTER_2 = "Q2";
  private static final String QUARTER_3 = "Q3";
  private static final String QUARTER_4 = "Q4";

  private VM member1;
  private VM member2;
  private VM member3;
  private VM member4;

  @Rule
  public DistributedRule distributedRule = new DistributedRule();
  @Rule
  public DistributedReference<Cache> cache = new DistributedReference<>();
  @Rule
  public DistributedDiskDirRule diskDirRule = new DistributedDiskDirRule();
  @Rule
  public DistributedRestoreSystemProperties restoreProps = new DistributedRestoreSystemProperties();
  @Rule
  public DistributedErrorCollector errorCollector = new DistributedErrorCollector();
  @Rule
  public DistributedBlackboard blackboard = new DistributedBlackboard();

  @Before
  public void setUp() {
    member1 = getVM(0);
    member2 = getVM(1);
    member3 = getVM(2);
    member4 = getVM(3);
  }

  @After
  public void tearDown() {
    for (VM vm : asList(getVM(0), getVM(1), getVM(2), getVM(3))) {
      vm.invoke(() -> {
        PartitionedRegion.BEFORE_CALCULATE_STARTING_BUCKET_FLAG = false;
        PartitionedRegionObserverHolder.clear();
      });
    }
  }

  /**
   * This test validates that null partition name cannot be added in FixedPartitionAttributes
   */
  @Test
  public void testNullPartitionName() {
    member1.invoke(() -> {
      cache.set(new CacheFactory(getDistributedSystemProperties()).create());
      FixedPartitionAttributes fpa = createFixedPartition(null, true, 3);

      Throwable thrown = catchThrowable(() -> {
        createPartitionedRegion(REGION_NAME, singletonList(fpa), 0, 40, 3,
            newQuarterPartitionResolver());
      });

      assertThat(thrown)
          .isInstanceOf(IllegalStateException.class)
          .hasMessageContaining("Fixed partition name cannot be null");
    });
  }

  /**
   * This tests validate that same partition name cannot be added more than once as primary as well
   * as secondary on same member.
   */
  @Test
  public void testSamePartitionNameTwice() {
    member1.invoke(() -> {
      cache.set(new CacheFactory(getDistributedSystemProperties()).create());
      FixedPartitionAttributes fpa1 = createFixedPartition(QUARTER_1, true, 3);
      FixedPartitionAttributes fpa2 = createFixedPartition(QUARTER_1, false, 3);

      Throwable thrown = catchThrowable(() -> {
        createPartitionedRegion(REGION_NAME, asList(fpa1, fpa2), 0, 40, 3,
            newQuarterPartitionResolver());
      });

      assertThat(thrown)
          .isInstanceOf(IllegalStateException.class)
          .hasMessageContaining("can be added only once in FixedPartitionAttributes");
    });
  }

  /**
   * This test validates that FixedPartitionAttributes cannot be defined for accessor nodes
   */
  @Test
  public void testFixedPartitionAttributes_Accessor() {
    member1.invoke(() -> {
      cache.set(new CacheFactory(getDistributedSystemProperties()).create());
      FixedPartitionAttributes fpa1 = createFixedPartition(QUARTER_1, true, 3);
      FixedPartitionAttributes fpa2 = createFixedPartition(QUARTER_2, false, 3);

      Throwable thrown = catchThrowable(() -> {
        createPartitionedRegion(REGION_NAME, asList(fpa1, fpa2), 0, 0, 3,
            newQuarterPartitionResolver());
      });

      assertThat(thrown)
          .isInstanceOf(IllegalStateException.class)
          .hasMessageContaining("can not be defined for accessor");
    });
  }

  /**
   * Test validation : only one node should return primary for a particular partition name for a
   * specific FPR at any given point of time. DuplicatePrimaryPartitionException is thrown during
   * FPR creation if this condition is not met.
   */
  @Test
  public void testSamePartitionName_Primary_OnTwoMembers() {
    addIgnoredException(DuplicatePrimaryPartitionException.class);

    member1.invoke(() -> {
      cache.set(new CacheFactory(getDistributedSystemProperties()).create());
      FixedPartitionAttributes fpa = createFixedPartition(QUARTER_1, true, 3);
      createPartitionedRegion(REGION_NAME, singletonList(fpa), 0, 40, 9,
          newQuarterPartitionResolver());
    });

    member2.invoke(() -> {
      cache.set(new CacheFactory(getDistributedSystemProperties()).create());
      FixedPartitionAttributes fpa = createFixedPartition(QUARTER_2, true, 3);
      createPartitionedRegion(REGION_NAME, singletonList(fpa), 0, 40, 9,
          newQuarterPartitionResolver());
    });

    member3.invoke(() -> {
      cache.set(new CacheFactory(getDistributedSystemProperties()).create());
      FixedPartitionAttributes fpa = createFixedPartition(QUARTER_2, true, 3);

      Throwable thrown = catchThrowable(() -> {
        createPartitionedRegion(REGION_NAME, singletonList(fpa), 0, 40, 9,
            newQuarterPartitionResolver());
      });

      assertThat(thrown)
          .isInstanceOf(DuplicatePrimaryPartitionException.class)
          .hasMessageContaining("can not be defined as primary on more than one node");
    });
  }

  /**
   * Test validation : if same partition is having different num-buckets across the nodes then
   * illegalStateException will be thrown
   */
  @Test
  public void testSamePartitionName_DifferentNumBuckets() {
    addIgnoredException(IllegalStateException.class);

    member1.invoke(() -> {
      cache.set(new CacheFactory(getDistributedSystemProperties()).create());
      FixedPartitionAttributes fpa1 = createFixedPartition(QUARTER_1, true, 3);
      FixedPartitionAttributes fpa2 = createFixedPartition(QUARTER_2, false, 3);
      createPartitionedRegion(REGION_NAME, asList(fpa1, fpa2), 1, 40, 9,
          newQuarterPartitionResolver());
    });

    member2.invoke(() -> {
      cache.set(new CacheFactory(getDistributedSystemProperties()).create());
      FixedPartitionAttributes fpa1 = createFixedPartition(QUARTER_2, true, 3);
      FixedPartitionAttributes fpa2 = createFixedPartition(QUARTER_1, false, 8);

      Throwable thrown = catchThrowable(() -> {
        createPartitionedRegion(REGION_NAME, asList(fpa1, fpa2), 1, 40, 9,
            newQuarterPartitionResolver());
      });

      assertThat(thrown)
          .isInstanceOf(IllegalStateException.class)
          .hasMessageContaining("num-buckets are not same");
    });
  }

  /**
   * Number of primary partitions (which should be one for a partition) and secondary partitions of
   * a FPR for a partition should never exceed number of redundant copies + 1. IllegalStateException
   * is thrown during FPR creation if this condition is not met.
   */
  @Test
  public void testNumberOfPartitions() {
    addIgnoredException(IllegalStateException.class);

    member1.invoke(() -> {
      cache.set(new CacheFactory(getDistributedSystemProperties()).create());
      createPartitionedRegion(REGION_NAME, null, 1, 0, 9, newQuarterPartitionResolver());
    });

    member2.invoke(() -> {
      cache.set(new CacheFactory(getDistributedSystemProperties()).create());
      FixedPartitionAttributes fpa1 = createFixedPartition("Q11", true, 3);
      FixedPartitionAttributes fpa2 = createFixedPartition("Q12", false, 3);
      createPartitionedRegion(REGION_NAME, asList(fpa1, fpa2), 1, 40, 9,
          newQuarterPartitionResolver());
    });

    member3.invoke(() -> {
      cache.set(new CacheFactory(getDistributedSystemProperties()).create());
      FixedPartitionAttributes fpa1 = createFixedPartition("Q12", true, 3);
      FixedPartitionAttributes fpa2 = createFixedPartition("Q13", false, 3);
      FixedPartitionAttributes fpa3 = createFixedPartition("Q11", false, 3);
      createPartitionedRegion(REGION_NAME, asList(fpa1, fpa2, fpa3), 1, 40, 9,
          newQuarterPartitionResolver());
    });

    member4.invoke(() -> {
      cache.set(new CacheFactory(getDistributedSystemProperties()).create());
      FixedPartitionAttributes fpa1 = createFixedPartition("Q13", true, 3);
      FixedPartitionAttributes fpa2 = createFixedPartition("Q11", false, 3);

      Throwable thrown = catchThrowable(() -> {
        createPartitionedRegion(REGION_NAME, asList(fpa1, fpa2), 1, 40, 9,
            newQuarterPartitionResolver());
      });

      assertThat(thrown)
          .isInstanceOf(IllegalStateException.class)
          .hasMessageContaining("should never exceed number of redundant copies");
    });
  }

  /**
   * Sum of num-buckets for different primary partitions should not be greater than totalNumBuckets.
   */
  @Test
  public void testNumBuckets_totalNumBuckets() {
    addIgnoredException(IllegalStateException.class);

    member1.invoke(() -> {
      cache.set(new CacheFactory(getDistributedSystemProperties()).create());
      createPartitionedRegion(REGION_NAME, null, 1, 0, 5, newQuarterPartitionResolver());
    });

    member2.invoke(() -> {
      cache.set(new CacheFactory(getDistributedSystemProperties()).create());
      FixedPartitionAttributes fpa1 = createFixedPartition(QUARTER_1, true, 3);
      FixedPartitionAttributes fpa2 = createFixedPartition(QUARTER_2, false, 3);
      createPartitionedRegion(REGION_NAME, asList(fpa1, fpa2), 1, 40, 5,
          newQuarterPartitionResolver());
    });

    member3.invoke(() -> {
      cache.set(new CacheFactory(getDistributedSystemProperties()).create());
      FixedPartitionAttributes fpa1 = createFixedPartition(QUARTER_2, true, 3);
      FixedPartitionAttributes fpa2 = createFixedPartition(QUARTER_3, false, 3);

      Throwable thrown = catchThrowable(() -> {
        createPartitionedRegion(REGION_NAME, asList(fpa1, fpa2), 1, 40, 5,
            newQuarterPartitionResolver());
      });

      assertThat(thrown)
          .isInstanceOf(IllegalStateException.class)
          .hasMessageContaining(
              "for different primary partitions should not be greater than total-num-buckets");
    });
  }

  /**
   * This test validates that if the required partition is not available at the time of entry
   * operation then PartitionNotAvailableException is thrown
   */
  @Test
  public void testPut_PartitionNotAvailableException() {
    member1.invoke(() -> {
      cache.set(new CacheFactory(getDistributedSystemProperties()).create());
      createPartitionedRegion(REGION_NAME, null, 1, 0, 12, newQuarterPartitionResolver());
    });

    member2.invoke(() -> {
      cache.set(new CacheFactory(getDistributedSystemProperties()).create());
      FixedPartitionAttributes fpa = createFixedPartition(QUARTER_1, true, 3);
      createPartitionedRegion(REGION_NAME, singletonList(fpa), 1, 40, 12,
          newQuarterPartitionResolver());
    });

    member3.invoke(() -> {
      cache.set(new CacheFactory(getDistributedSystemProperties()).create());
      FixedPartitionAttributes fpa = createFixedPartition(QUARTER_2, true, 3);
      createPartitionedRegion(REGION_NAME, singletonList(fpa), 1, 40, 12,
          newQuarterPartitionResolver());
    });

    member1.invoke(() -> {
      Throwable thrown = catchThrowable(() -> {
        putThroughAccessor(REGION_NAME);
      });

      assertThat(thrown)
          .isInstanceOf(PartitionNotAvailableException.class);
    });
  }

  /**
   * This test validates that if one datastore has the fixed partition attributes defined then other
   * datastore should also have the fixed partition attributes defined
   */
  @Test
  public void test_DataStoreWithoutPartition_DataStoreWithPartition() {
    addIgnoredException(IllegalStateException.class);

    member1.invoke(() -> {
      cache.set(new CacheFactory(getDistributedSystemProperties()).create());
      createPartitionedRegion(REGION_NAME, null, 1, 40, 12, newQuarterPartitionResolver());
    });

    member2.invoke(() -> {
      cache.set(new CacheFactory(getDistributedSystemProperties()).create());
      FixedPartitionAttributes fpa = createFixedPartition(QUARTER_1, true, 3);

      Throwable thrown = catchThrowable(() -> {
        createPartitionedRegion(REGION_NAME, singletonList(fpa), 1, 40, 12,
            newQuarterPartitionResolver());
      });

      assertThat(thrown)
          .isInstanceOf(IllegalStateException.class);
    });
  }

  /**
   * This test validates that if one datastore has the fixed partition attributes defined then other
   * datastore should also have the fixed partition attributes defined
   */
  @Test
  public void test_DataStoreWithPartition_DataStoreWithoutPartition() {
    addIgnoredException(IllegalStateException.class);

    member2.invoke(() -> {
      cache.set(new CacheFactory(getDistributedSystemProperties()).create());
      FixedPartitionAttributes fpa = createFixedPartition(QUARTER_1, true, 3);
      createPartitionedRegion(REGION_NAME, singletonList(fpa), 1, 40, 12,
          newQuarterPartitionResolver());
    });

    member1.invoke(() -> {
      cache.set(new CacheFactory(getDistributedSystemProperties()).create());

      Throwable thrown = catchThrowable(() -> {
        createPartitionedRegion(REGION_NAME, null, 1, 40, 12, newQuarterPartitionResolver());
      });

      assertThat(thrown)
          .isInstanceOf(IllegalStateException.class);
    });
  }

  /**
   * This tests validate that accessor member does the put on datastores as per primary
   * FixedPartitionAttributes defined on datastores
   */
  @Test
  public void testPut_ValidateDataOnMember_OnlyPrimary_Accessor() {
    member1.invoke(() -> {
      cache.set(new CacheFactory(getDistributedSystemProperties()).create());
      createPartitionedRegion(REGION_NAME, null, 0, 0, 12, newQuarterPartitionResolver());
    });

    member2.invoke(() -> {
      cache.set(new CacheFactory(getDistributedSystemProperties()).create());
      FixedPartitionAttributes fpa = createFixedPartition(QUARTER_1, true, 3);
      createPartitionedRegion(REGION_NAME, singletonList(fpa), 0, 40, 12,
          newQuarterPartitionResolver());
    });

    member3.invoke(() -> {
      cache.set(new CacheFactory(getDistributedSystemProperties()).create());
      FixedPartitionAttributes fpa = createFixedPartition(QUARTER_2, true, 3);
      createPartitionedRegion(REGION_NAME, singletonList(fpa), 0, 40, 12,
          newQuarterPartitionResolver());
    });

    member4.invoke(() -> {
      cache.set(new CacheFactory(getDistributedSystemProperties()).create());
      FixedPartitionAttributes fpa = createFixedPartition(QUARTER_3, true, 3);
      createPartitionedRegion(REGION_NAME, singletonList(fpa), 0, 40, 12,
          newQuarterPartitionResolver());
    });

    member1.invoke(() -> putThroughAccessor(REGION_NAME));

    member2.invoke(() -> checkPrimaryData(QUARTER_1));
    member3.invoke(() -> checkPrimaryData(QUARTER_2));
    member4.invoke(() -> checkPrimaryData(QUARTER_3));
  }

  @Test
  public void putWhileDataStoresAreBeingCreatedFails() throws Exception {
    VM accessor = getVM(0);
    VM datastore1 = getVM(1);
    VM datastore2 = getVM(2);
    VM datastore3 = getVM(3);

    accessor.invoke(() -> cache.set(new CacheFactory(getDistributedSystemProperties()).create()));
    datastore1.invoke(() -> cache.set(new CacheFactory(getDistributedSystemProperties()).create()));
    datastore2.invoke(() -> cache.set(new CacheFactory(getDistributedSystemProperties()).create()));
    datastore3.invoke(() -> cache.set(new CacheFactory(getDistributedSystemProperties()).create()));

    blackboard.initBlackboard();

    // Create an accessor for the PR
    accessor.invoke(
        () -> createPartitionedRegion(REGION_NAME, null, 0, 0, 12, newQuarterPartitionResolver()));

    // Set an observer in datastore 1 that will wait in the middle of PR creation
    datastore1.invoke(this::setPRObserverBeforeCalculateStartingBucketId);

    // Start datastore1 asynchronously. This will get stuck waiting in the PR observer
    AsyncInvocation<Void> createRegionInDatastore1 =
        datastore1.invokeAsync(() -> createRegionWithQuarter(QUARTER_1));

    // Make sure datastore 1 gets stuck waiting on the observer
    blackboard.waitForGate("waiting");

    // Start two more data stores asynchronously. These will get stuck waiting for a dlock behind
    // datastore1
    AsyncInvocation<Void> createRegionInDatastore2 =
        datastore2.invokeAsync(() -> createRegionWithQuarter(QUARTER_2));
    AsyncInvocation<Void> createRegionInDatastore3 =
        datastore3.invokeAsync(() -> createRegionWithQuarter(QUARTER_3));

    // Make sure a put to datastore1, which is in the middle of pr initialization, fails correctly
    accessor.invoke(() -> {
      Throwable thrown = catchThrowable(() -> {
        putForQuarter(REGION_NAME, QUARTER_1);
      });

      assertThat(thrown)
          .isInstanceOf(PartitionedRegionStorageException.class);
    });

    // Make sure a put to datastore2, which stuck behind datastore1 in initialization, fails
    // correctly
    accessor.invoke(() -> {
      Throwable thrown = catchThrowable(() -> {
        putForQuarter(REGION_NAME, QUARTER_2);
      });

      assertThat(thrown)
          .isInstanceOf(PartitionedRegionStorageException.class);
    });

    blackboard.signalGate("done");

    createRegionInDatastore1.await();
    createRegionInDatastore2.await();
    createRegionInDatastore3.await();
  }

  /**
   * This tests validate that datastore member does the put on itself as well as other datastores as
   * per primary FixedPartitionAttributes defined on datastores.
   */
  @Test
  public void testPut_ValidateDataOnMember_OnlyPrimary_Datastore() {
    member1.invoke(() -> {
      cache.set(new CacheFactory(getDistributedSystemProperties()).create());
      FixedPartitionAttributes fpa = createFixedPartition(QUARTER_1, true, 3);
      createPartitionedRegion("Quarter", singletonList(fpa), 0, 40, 12,
          newQuarterPartitionResolver());
    });

    member2.invoke(() -> {
      cache.set(new CacheFactory(getDistributedSystemProperties()).create());
      FixedPartitionAttributes fpa = createFixedPartition(QUARTER_2, true, 3);
      createPartitionedRegion("Quarter", singletonList(fpa), 0, 40, 12,
          newQuarterPartitionResolver());
    });

    member3.invoke(() -> {
      cache.set(new CacheFactory(getDistributedSystemProperties()).create());
      FixedPartitionAttributes fpa = createFixedPartition(QUARTER_3, true, 3);
      createPartitionedRegion("Quarter", singletonList(fpa), 0, 40, 12,
          newQuarterPartitionResolver());
    });

    member4.invoke(() -> {
      cache.set(new CacheFactory(getDistributedSystemProperties()).create());
      FixedPartitionAttributes fpa = createFixedPartition(QUARTER_4, true, 3);
      createPartitionedRegion("Quarter", singletonList(fpa), 0, 40, 12,
          newQuarterPartitionResolver());
    });

    member1.invoke(() -> putThroughDataStore("Quarter"));

    member1.invoke(() -> checkPrimaryData(QUARTER_1));
    member2.invoke(() -> checkPrimaryData(QUARTER_2));
    member3.invoke(() -> checkPrimaryData(QUARTER_3));
    member4.invoke(() -> checkPrimaryData(QUARTER_4));

    member1.invoke(() -> checkPrimaryBucketsForQuarter(3, 3));
    member2.invoke(() -> checkPrimaryBucketsForQuarter(3, 3));
    member3.invoke(() -> checkPrimaryBucketsForQuarter(3, 3));
    member4.invoke(() -> checkPrimaryBucketsForQuarter(3, 3));
  }

  /**
   * This test validate that a delete operation on empty region will throw EntryNotFoundException
   */
  @Test
  public void testDelete_WithoutPut() {
    member1.invoke(() -> {
      cache.set(new CacheFactory(getDistributedSystemProperties()).create());
      FixedPartitionAttributes fpa = createFixedPartition(QUARTER_1, true, 3);
      createPartitionedRegion("Quarter", singletonList(fpa), 0, 40, 12,
          newQuarterPartitionResolver());
    });

    member2.invoke(() -> {
      cache.set(new CacheFactory(getDistributedSystemProperties()).create());
      FixedPartitionAttributes fpa = createFixedPartition(QUARTER_2, true, 3);
      createPartitionedRegion("Quarter", singletonList(fpa), 0, 40, 12,
          newQuarterPartitionResolver());
    });

    member3.invoke(() -> {
      cache.set(new CacheFactory(getDistributedSystemProperties()).create());
      FixedPartitionAttributes fpa = createFixedPartition(QUARTER_3, true, 3);
      createPartitionedRegion("Quarter", singletonList(fpa), 0, 40, 12,
          newQuarterPartitionResolver());
    });

    member4.invoke(() -> {
      cache.set(new CacheFactory(getDistributedSystemProperties()).create());
      FixedPartitionAttributes fpa = createFixedPartition(QUARTER_4, true, 3);
      createPartitionedRegion("Quarter", singletonList(fpa), 0, 40, 12,
          newQuarterPartitionResolver());
    });

    member1.invoke(() -> {
      Throwable thrown = catchThrowable(() -> {
        deleteOperation("Quarter");
      });

      assertThat(thrown)
          .isInstanceOf(EntryNotFoundException.class);

      putThroughDataStore("Quarter");
      getThroughDataStore("Quarter");
    });
  }

  /**
   * This tests validate that datastore member tries the put on itself as well as other datastores
   * as per primary FixedPartitionAttributes defined on datastores. But No resolver is provided. So
   * IllegalStateException in expected
   */
  @Test
  public void testPut_NoResolver() {
    member1.invoke(() -> {
      cache.set(new CacheFactory(getDistributedSystemProperties()).create());
      FixedPartitionAttributes fpa = createFixedPartition(QUARTER_1, true, 3);
      createPartitionedRegion("Quarter", singletonList(fpa), 0, 40, 12, null);
    });

    member2.invoke(() -> {
      cache.set(new CacheFactory(getDistributedSystemProperties()).create());
      FixedPartitionAttributes fpa = createFixedPartition(QUARTER_2, true, 3);
      createPartitionedRegion("Quarter", singletonList(fpa), 0, 40, 12, null);
    });

    member3.invoke(() -> {
      cache.set(new CacheFactory(getDistributedSystemProperties()).create());
      FixedPartitionAttributes fpa = createFixedPartition(QUARTER_3, true, 3);
      createPartitionedRegion("Quarter", singletonList(fpa), 0, 40, 12, null);
    });

    member4.invoke(() -> {
      cache.set(new CacheFactory(getDistributedSystemProperties()).create());
      FixedPartitionAttributes fpa = createFixedPartition(QUARTER_4, true, 3);
      createPartitionedRegion("Quarter", singletonList(fpa), 0, 40, 12, null);
    });

    member1.invoke(() -> {
      Throwable thrown = catchThrowable(() -> {
        putThroughDataStore_NoResolver("Quarter");
      });

      assertThat(thrown)
          .isInstanceOf(IllegalStateException.class);
    });
  }

  /**
   * This tests validate that datastore member tries the put with callback on itself as well as
   * other datastores as per primary FixedPartitionAttributes defined on datastores. here CallBack
   * implements FixedPartitionResolver.
   */
  @Test
  public void testPut_CallBackWithResolver() {
    Properties props = getDistributedSystemProperties();
    props.setProperty(SERIALIZABLE_OBJECT_FILTER, "*");

    member1.invoke(() -> {
      cache.set(new CacheFactory(props).create());
      FixedPartitionAttributes fpa = createFixedPartition(QUARTER_1, true, 3);
      createPartitionedRegion("Quarter", singletonList(fpa), 0, 40, 12, null);
    });

    member2.invoke(() -> {
      cache.set(new CacheFactory(props).create());
      FixedPartitionAttributes fpa = createFixedPartition(QUARTER_2, true, 3);
      createPartitionedRegion("Quarter", singletonList(fpa), 0, 40, 12, null);
    });

    member3.invoke(() -> {
      cache.set(new CacheFactory(props).create());
      FixedPartitionAttributes fpa = createFixedPartition(QUARTER_3, true, 3);
      createPartitionedRegion("Quarter", singletonList(fpa), 0, 40, 12, null);
    });

    member4.invoke(() -> {
      cache.set(new CacheFactory(props).create());
      FixedPartitionAttributes fpa = createFixedPartition(QUARTER_4, true, 3);
      createPartitionedRegion("Quarter", singletonList(fpa), 0, 40, 12, null);
    });

    member1.invoke(() -> putThroughDataStore_CallBackWithResolver("Quarter"));

    member1.invoke(() -> checkPrimaryData(QUARTER_1));
    member2.invoke(() -> checkPrimaryData(QUARTER_2));
    member3.invoke(() -> checkPrimaryData(QUARTER_3));
    member4.invoke(() -> checkPrimaryData(QUARTER_4));

    member1.invoke(() -> checkPrimaryBucketsForQuarter(3, 3));
    member2.invoke(() -> checkPrimaryBucketsForQuarter(3, 3));
    member3.invoke(() -> checkPrimaryBucketsForQuarter(3, 3));
    member4.invoke(() -> checkPrimaryBucketsForQuarter(3, 3));
  }

  /**
   * This test validates that a PR without Fixed Partition Attributes and with
   * FixedPartitionResolver will do custom partitioning as per resolver.
   */
  @Test
  public void testPut_WithResolver_NoFPAs() {
    member1.invoke(() -> {
      cache.set(new CacheFactory(getDistributedSystemProperties()).create());
      createPartitionedRegion("Quarter", null, 0, 40, 12, newQuarterPartitionResolver());
    });

    member2.invoke(() -> {
      cache.set(new CacheFactory(getDistributedSystemProperties()).create());
      createPartitionedRegion("Quarter", null, 0, 40, 12, newQuarterPartitionResolver());
    });

    member3.invoke(() -> {
      cache.set(new CacheFactory(getDistributedSystemProperties()).create());
      createPartitionedRegion("Quarter", null, 0, 40, 12, newQuarterPartitionResolver());
    });

    member4.invoke(() -> {
      cache.set(new CacheFactory(getDistributedSystemProperties()).create());
      createPartitionedRegion("Quarter", null, 0, 40, 12, newQuarterPartitionResolver());
    });

    member1.invoke(() -> putThroughDataStore("Quarter"));
  }

  /**
   * This tests validate that datastore member tries the put on itself as well as other datastores
   * as per primary FixedPartitionAttributes defined on datastores. Here No Resolver is provided
   * through attributes. Some keys implement FixedPartitionResolver and some does't implement any
   * resolver. IllegalStateException is expected.
   */
  @Test
  public void testPut_FixedPartitionResolver_NoResolver() {
    member1.invoke(() -> {
      cache.set(new CacheFactory(getDistributedSystemProperties()).create());
      FixedPartitionAttributes fpa = createFixedPartition(QUARTER_1, true, 3);
      createPartitionedRegion("Quarter", singletonList(fpa), 0, 40, 12, null);
    });

    member2.invoke(() -> {
      cache.set(new CacheFactory(getDistributedSystemProperties()).create());
      FixedPartitionAttributes fpa = createFixedPartition(QUARTER_2, true, 3);
      createPartitionedRegion("Quarter", singletonList(fpa), 0, 40, 12, null);
    });

    member3.invoke(() -> {
      cache.set(new CacheFactory(getDistributedSystemProperties()).create());
      FixedPartitionAttributes fpa = createFixedPartition(QUARTER_3, true, 3);
      createPartitionedRegion("Quarter", singletonList(fpa), 0, 40, 12, null);
    });

    member4.invoke(() -> {
      cache.set(new CacheFactory(getDistributedSystemProperties()).create());
      FixedPartitionAttributes fpa = createFixedPartition(QUARTER_4, true, 3);
      createPartitionedRegion("Quarter", singletonList(fpa), 0, 40, 12, null);
    });

    member1.invoke(() -> {
      Throwable thrown = catchThrowable(() -> {
        putThroughDataStore_FixedPartitionResolver_NoResolver("Quarter");
      });

      assertThat(thrown)
          .isInstanceOf(IllegalStateException.class);
    });
  }

  /**
   * This tests validate that datastore member tries the put on itself as well as other datastores
   * as per primary FixedPartitionAttributes defined on datastores. Here No Resolver is provided
   * through attributes. Some keys implement FixedPartitionResolver and some implements
   * PartitionResolver. IllegalStateException is expected.
   */
  @Test
  public void testPut_FixedPartitionResolver_PartitionResolver() {
    member1.invoke(() -> {
      cache.set(new CacheFactory(getDistributedSystemProperties()).create());
      FixedPartitionAttributes fpa = createFixedPartition(QUARTER_1, true, 3);
      createPartitionedRegion("Quarter", singletonList(fpa), 0, 40, 12, null);
    });

    member2.invoke(() -> {
      cache.set(new CacheFactory(getDistributedSystemProperties()).create());
      FixedPartitionAttributes fpa = createFixedPartition(QUARTER_2, true, 3);
      createPartitionedRegion("Quarter", singletonList(fpa), 0, 40, 12, null);
    });

    member3.invoke(() -> {
