/*
 * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
 * agreements. See the NOTICE file distributed with this work for additional information regarding
 * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance with the License. You may obtain a
 * copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

/**
 * Test various distributed aspects of transactions, e.g. locking/reservation symantics that do not
 * need multiple Region configurations. For those tests see <code>MultiVMRegionTestCase</code>.
 *
 *
 * @since GemFire 4.0
 * @see MultiVMRegionTestCase
 *
 */
package org.apache.geode.cache30;

import static org.apache.geode.distributed.ConfigurationProperties.ENABLE_NETWORK_PARTITION_DETECTION;
import static org.apache.geode.distributed.ConfigurationProperties.LOG_LEVEL;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;

import java.io.Serializable;
import java.util.HashSet;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.CountDownLatch;

import org.junit.Ignore;
import org.junit.Test;

import org.apache.geode.SystemFailure;
import org.apache.geode.cache.AttributesFactory;
import org.apache.geode.cache.Cache;
import org.apache.geode.cache.CacheException;
import org.apache.geode.cache.CacheLoader;
import org.apache.geode.cache.CacheTransactionManager;
import org.apache.geode.cache.CommitConflictException;
import org.apache.geode.cache.CommitIncompleteException;
import org.apache.geode.cache.DataPolicy;
import org.apache.geode.cache.DiskAccessException;
import org.apache.geode.cache.LoaderHelper;
import org.apache.geode.cache.MirrorType;
import org.apache.geode.cache.Operation;
import org.apache.geode.cache.Region;
import org.apache.geode.cache.RegionAttributes;
import org.apache.geode.cache.Scope;
import org.apache.geode.distributed.internal.ResourceEvent;
import org.apache.geode.distributed.internal.ResourceEventsListener;
import org.apache.geode.distributed.internal.locks.DLockBatch;
import org.apache.geode.distributed.internal.locks.DLockService;
import org.apache.geode.distributed.internal.membership.InternalDistributedMember;
import org.apache.geode.distributed.internal.membership.api.MembershipManagerHelper;
import org.apache.geode.internal.cache.CommitReplyException;
import org.apache.geode.internal.cache.GemFireCacheImpl;
import org.apache.geode.internal.cache.InternalRegionFactory;
import org.apache.geode.internal.cache.LocalRegion;
import org.apache.geode.internal.cache.RegionEntry;
import org.apache.geode.internal.cache.TXManagerImpl;
import org.apache.geode.internal.cache.TXState;
import org.apache.geode.internal.cache.TXStateInterface;
import org.apache.geode.internal.cache.TXStateProxyImpl;
import org.apache.geode.internal.cache.locks.TXLockBatch;
import org.apache.geode.internal.cache.locks.TXLockService;
import org.apache.geode.internal.cache.locks.TXLockServiceImpl;
import org.apache.geode.test.awaitility.GeodeAwaitility;
import org.apache.geode.test.dunit.Assert;
import org.apache.geode.test.dunit.Host;
import org.apache.geode.test.dunit.IgnoredException;
import org.apache.geode.test.dunit.Invoke;
import org.apache.geode.test.dunit.LogWriterUtils;
import org.apache.geode.test.dunit.SerializableCallable;
import org.apache.geode.test.dunit.SerializableRunnable;
import org.apache.geode.test.dunit.VM;
import org.apache.geode.test.dunit.WaitCriterion;
import org.apache.geode.test.dunit.cache.internal.JUnit4CacheTestCase;


public class TXDistributedDUnitTest extends JUnit4CacheTestCase {

  protected <K, V> RegionAttributes<K, V> getRegionAttributes() {
    return getRegionAttributes(Scope.DISTRIBUTED_ACK);
  }

  protected <K, V> RegionAttributes<K, V> getRegionAttributes(Scope scope) {
    AttributesFactory<K, V> factory = new AttributesFactory<>();
    factory.setScope(scope);
    if (scope.isDistributedAck()) {
      factory.setEarlyAck(false);
    }
    return factory.create();
  }

  /**
   * Test a remote grantor
   */
  @Test
  public void testRemoteGrantor() throws Exception {
    IgnoredException.addIgnoredException("killing members ds");
    final CacheTransactionManager txMgr = getCache().getCacheTransactionManager();
    final String rgnName = getUniqueName();
    Region rgn = getCache().createRegion(rgnName, getRegionAttributes());
    rgn.create("key", null);

    Invoke.invokeInEveryVM(new SerializableRunnable("testRemoteGrantor: initial configuration") {
      @Override
      public void run() {
        try {
          Region rgn1 = getCache().createRegion(rgnName, getRegionAttributes());
          rgn1.put("key", "val0");
        } catch (CacheException e) {
          Assert.fail("While creating region", e);
        }
      }
    });

    Host host = Host.getHost(0);
    VM vm0 = host.getVM(0);
    // VM vm1 = host.getVM(1);
    // VM vm2 = host.getVM(2);

    vm0.invoke(new SerializableRunnable("testRemoteGrantor: remote grantor init") {
      @Override
      public void run() {
        try {
          Region rgn1 = getCache().getRegion(rgnName);
          final CacheTransactionManager txMgr2 = getCache().getCacheTransactionManager();
          txMgr2.begin();
          rgn1.put("key", "val1");
          txMgr2.commit();
          assertNotNull(TXLockService.getDTLS());
          assertTrue(TXLockService.getDTLS().isLockGrantor());
        } catch (CacheException e) {
          fail("While performing first transaction");
        }
      }
    });

    // fix for bug 38843 causes the DTLS to be created in every TX participant
    assertNotNull(TXLockService.getDTLS());
    assertFalse(TXLockService.getDTLS().isLockGrantor());
    assertEquals("val1", rgn.getEntry("key").getValue());

    vm0.invoke(new SerializableRunnable("Disconnect from DS, remote grantor death") {
      @Override
      public void run() {
        try {
          MembershipManagerHelper.crashDistributedSystem(getSystem());
        } finally {
          // Allow getCache() to re-establish a ds connection
          closeCache();
        }
      }
    });

    // Make this VM the remote Grantor
    txMgr.begin();
    rgn.put("key", "val2");
    txMgr.commit();
    assertNotNull(TXLockService.getDTLS());
    assertTrue(TXLockService.getDTLS().isLockGrantor());

    SerializableRunnable remoteComm =
        new SerializableRunnable("testRemoteGrantor: remote grantor commit") {
          @Override
          public void run() {
            try {
              Cache c = getCache();
              CacheTransactionManager txMgr2 = c.getCacheTransactionManager();
              Region rgn1 = c.getRegion(rgnName);
              if (rgn1 == null) {
                // This block should only execute on VM0
                rgn1 = c.createRegion(rgnName, getRegionAttributes());
              }

              txMgr2.begin();
              rgn1.put("key", "val3");
              txMgr2.commit();

              if (TXLockService.getDTLS() != null) {
                assertTrue(!TXLockService.getDTLS().isLockGrantor());
              }
            } catch (CacheException e) {
              Assert.fail("While creating region", e);
            }
          }
        };
    Invoke.invokeInEveryVM(remoteComm);
    // vm1.invoke(remoteComm);
    // vm2.invoke(remoteComm);

    assertNotNull(TXLockService.getDTLS());
    assertTrue(TXLockService.getDTLS().isLockGrantor());
    assertEquals("val3", rgn.getEntry("key").getValue());
    rgn.destroyRegion();
  }

  /**
   * Test the internal callbacks used for what else... testing
   */
  @Test
  public void testInternalCallbacks() throws Exception {
    final CacheTransactionManager txMgr = getCache().getCacheTransactionManager();
    final String rgnName1 = getUniqueName() + "_1";
    final String rgnName2 = getUniqueName() + "_2";
    final String rgnName3 = getUniqueName() + "_3";
    Region rgn1 = getCache().createRegion(rgnName1, getRegionAttributes());

    Host host = Host.getHost(0);
    VM vm0 = host.getVM(0);
    VM vm1 = host.getVM(1);

    SerializableRunnable createRgn =
        new SerializableRunnable("testInternalCallbacks: initial configuration") {
          @Override
          public void run() {
            try {
              Region rgn1a = getCache().createRegion(rgnName1, getRegionAttributes());
              Region rgn2 = getCache().createRegion(rgnName2, getRegionAttributes());
              Region rgn3 =
                  getCache().createRegion(rgnName3, getRegionAttributes(Scope.DISTRIBUTED_NO_ACK));
              rgn1a.create("key", null);
              rgn2.create("key", null);
              rgn3.create("key", null);
            } catch (CacheException e) {
              Assert.fail("While creating region", e);
            }
          }
        };
    vm0.invoke(createRgn);
    vm1.invoke(createRgn);

    // Standard commit check
    txMgr.begin();
    rgn1.put("key", "value0");
    txMgr.commit();
    SerializableRunnable checkRgn1 =
        new SerializableRunnable("testInternalCallbacks: check rgn1 valus") {
          @Override
          public void run() {
            Region rgn1a = getCache().getRegion(rgnName1);
            assertNotNull(rgn1a);
            assertEquals("value0", rgn1a.getEntry("key").getValue());
          }
        };
    vm0.invoke(checkRgn1);
    vm1.invoke(checkRgn1);

    {
      final byte[] cbSensors = {0, 0, 0, 0, 0, 0, 0, 0, 0};
      txMgr.begin();
      ((TXStateProxyImpl) ((TXManagerImpl) txMgr).getTXState()).forceLocalBootstrap();
      setInternalCallbacks(((TXManagerImpl) txMgr).getTXState(), cbSensors);
      rgn1.put("key", "value1");
      txMgr.commit();
      for (int i = cbSensors.length - 3; i >= 0; --i) {
        assertEquals("Internal callback " + i + " was not called the expected number of times!",
            (byte) 1, cbSensors[i]);
      }
      for (int i = cbSensors.length - 1; i > cbSensors.length - 3; --i) {
        assertEquals(
            "Internal \"during\" callback " + i + " invoked an unexpected number of times!",
            (byte) 2, cbSensors[i]);
      }
    }
    SerializableRunnable checkRgn1Again =
        new SerializableRunnable("testInternalCallbacks: validate remote values") {
          @Override
          public void run() {
            Region rgn1a = getCache().getRegion(rgnName1);
            assertNotNull(rgn1a);
            assertEquals("value1", rgn1a.getEntry("key").getValue());
          }
        };
    vm0.invoke(checkRgn1Again);
    vm1.invoke(checkRgn1Again);

    // Try 2 regions
    Region rgn2 = getCache().createRegion(rgnName2, getRegionAttributes());
    txMgr.begin();
    rgn1.put("key", "value2");
    rgn2.put("key", "value2");
    txMgr.commit();
    SerializableRunnable checkRgn12 =
        new SerializableRunnable("testInternalCallbacks: check rgn1 valus") {
          @Override
          public void run() {
            Region rgn1a = getCache().getRegion(rgnName1);
            assertNotNull(rgn1a);
            assertEquals("value2", rgn1a.getEntry("key").getValue());
            Region rgn2a = getCache().getRegion(rgnName2);
            assertNotNull(rgn2a);
            assertEquals("value2", rgn2a.getEntry("key").getValue());
          }
        };
    vm0.invoke(checkRgn12);
    vm1.invoke(checkRgn12);

    {
      final byte[] cbSensors = {0, 0, 0, 0, 0, 0, 0, 0, 0};
      txMgr.begin();
      ((TXStateProxyImpl) ((TXManagerImpl) txMgr).getTXState()).forceLocalBootstrap();
      setInternalCallbacks(((TXManagerImpl) txMgr).getTXState(), cbSensors);
      rgn1.put("key", "value3");
      rgn2.put("key", "value3");
      txMgr.commit();

      for (int i = cbSensors.length - 3; i >= 0; i--) {
        assertEquals("Internal callback " + i + " was not called the expected number of times!",
            (byte) 1, cbSensors[i]);
      }
      for (int i = cbSensors.length - 1; i > cbSensors.length - 3; --i) {
        assertEquals(
            "Internal \"during\" callback " + i + " invoked an unexpected number of times!",
            (byte) 2, cbSensors[i]);
      }
    }
    SerializableRunnable checkRgn12Again =
        new SerializableRunnable("testInternalCallbacks: validate both regions remote values") {
          @Override
          public void run() {
            Region rgn1a = getCache().getRegion(rgnName1);
            assertNotNull(rgn1a);
            assertEquals("value3", rgn1a.getEntry("key").getValue());
            Region rgn2a = getCache().getRegion(rgnName2);
            assertNotNull(rgn2a);
            assertEquals("value3", rgn2a.getEntry("key").getValue());
          }
        };
    vm0.invoke(checkRgn12Again);
    vm1.invoke(checkRgn12Again);

    // Try a third region (D_NO_ACK)
    Region rgn3 = getCache().createRegion(rgnName3, getRegionAttributes(Scope.DISTRIBUTED_NO_ACK));
    txMgr.begin();
    rgn1.put("key", "value4");
    rgn2.put("key", "value4");
    rgn3.put("key", "value4");
    txMgr.commit();
    SerializableRunnable checkRgn123 =
        new SerializableRunnable("testInternalCallbacks: check rgn1 valus") {
          @Override
          public void run() {
            Region rgn1a = getCache().getRegion(rgnName1);
            assertNotNull(rgn1a);
            assertEquals("value4", rgn1a.getEntry("key").getValue());
            Region rgn2a = getCache().getRegion(rgnName2);
            assertNotNull(rgn2a);
            assertEquals("value4", rgn2a.getEntry("key").getValue());
            Region rgn3a = getCache().getRegion(rgnName3);
            assertNotNull(rgn3a);
            assertEquals("value4", rgn3a.getEntry("key").getValue());
          }
        };
    vm0.invoke(checkRgn123);
    vm1.invoke(checkRgn123);

    {
      final byte[] cbSensors = {0, 0, 0, 0, 0, 0, 0, 0, 0};
      txMgr.begin();
      ((TXStateProxyImpl) ((TXManagerImpl) txMgr).getTXState()).forceLocalBootstrap();
      setInternalCallbacks(((TXManagerImpl) txMgr).getTXState(), cbSensors);

      rgn1.put("key", "value5");
      rgn2.put("key", "value5");
      rgn3.put("key", "value5");
      txMgr.commit();

      for (int i = cbSensors.length - 3; i >= 0; i--) {
        assertEquals("Internal callback " + i + " was not called the expected number of times!",
            (byte) 1, cbSensors[i]);
      }
      for (int i = cbSensors.length - 1; i > cbSensors.length - 3; --i) {
        assertEquals(
            "Internal \"during\" callback " + i + " invoked an unexpected number of times!",
            (byte) 2, cbSensors[i]);
      }
    }
    SerializableRunnable checkRgn123Again =
        new SerializableRunnable("testInternalCallbacks: validate both regions remote values") {
          @Override
          public void run() {
            Region rgn1a = getCache().getRegion(rgnName1);
            assertNotNull(rgn1a);
            assertEquals("value5", rgn1a.getEntry("key").getValue());
            Region rgn2a = getCache().getRegion(rgnName2);
            assertNotNull(rgn2a);
            assertEquals("value5", rgn2a.getEntry("key").getValue());
            Region rgn3a = getCache().getRegion(rgnName3);
            assertNotNull(rgn3a);
            assertEquals("value5", rgn3a.getEntry("key").getValue());
          }
        };
    vm0.invoke(checkRgn123Again);
    vm1.invoke(checkRgn123Again);

    rgn1.destroyRegion();
    rgn2.destroyRegion();
  }

  static void setInternalCallbacks(TXStateInterface txp, final byte[] cbSensors) {
    ((TXStateProxyImpl) txp).forceLocalBootstrap();
    TXState tx = (TXState) ((TXStateProxyImpl) txp).getRealDeal(null, null);
    assertEquals(9, cbSensors.length);
    tx.setAfterReservation(() -> cbSensors[0]++);
    tx.setAfterConflictCheck(() -> cbSensors[1]++);
    tx.setAfterApplyChanges(() -> cbSensors[2]++);
    tx.setAfterReleaseLocalLocks(() -> cbSensors[3]++);
    tx.setAfterIndividualSend(() -> cbSensors[4]++);
    tx.setAfterIndividualCommitProcess(() -> cbSensors[5]++);
    tx.setAfterSend(() -> cbSensors[6]++);
    tx.setDuringIndividualSend(() -> cbSensors[7]++);
    tx.setDuringIndividualCommitProcess(() -> cbSensors[8]++);
  }

  /**
   * Test distributed ack transactions that consist only of data from loaded values
   */
  @Test
  public void testDACKLoadedMessage() throws Exception {
    final CacheTransactionManager txMgr = getCache().getCacheTransactionManager();
    final String rgnName = getUniqueName();
    AttributesFactory factory = new AttributesFactory();
    factory.setScope(Scope.DISTRIBUTED_ACK);
    factory.setEarlyAck(false);
    factory.setCacheLoader(new CacheLoader() {
      @Override
      public Object load(LoaderHelper helper) {
        return "val" + helper.getArgument();
      }

      @Override
      public void close() {}
    });
    Region rgn = getCache().createRegion(rgnName, factory.create());

    Invoke.invokeInEveryVM(new SerializableRunnable("testDACKLoadedMessage: intial configuration") {
      @Override
      public void run() {
        try {
          AttributesFactory factory2 = new AttributesFactory();
          factory2.setScope(Scope.DISTRIBUTED_ACK);
          factory2.setEarlyAck(false);
          // factory.setDataPolicy(DataPolicy.REPLICATE);
          factory2.setMirrorType(MirrorType.KEYS);
          getCache().createRegion(rgnName, factory2.create());
        } catch (CacheException e) {
          Assert.fail("While creating region", e);
        }
      }
    });

    // Confirm the standard case
    txMgr.begin();
    rgn.put("key1", "val1");
    txMgr.commit();
    assertEquals("val1", rgn.getEntry("key1").getValue());

    Invoke
        .invokeInEveryVM(new SerializableRunnable("testDACKLoadedMessage: confirm standard case") {
          @Override
          public void run() {
            Region rgn1 = getCache().getRegion(rgnName);
            assertEquals("val1", rgn1.getEntry("key1").getValue());
          }
        });

    // Confirm loaded value case
    txMgr.begin();
    rgn.get("key2", 2);
    txMgr.commit();
    assertEquals("val2", rgn.getEntry("key2").getValue());

    Invoke
        .invokeInEveryVM(new SerializableRunnable("testDACKLoadedMessage: confirm standard case") {
          @Override
          public void run() {
            Region rgn1 = getCache().getRegion(rgnName);
            assertEquals("val2", rgn1.getEntry("key2").getValue());
          }
        });

    // This should use the ack w/ the lockid
    txMgr.begin();
    rgn.put("key3", "val3");
    rgn.get("key4", 4);
    txMgr.commit();

    Invoke
        .invokeInEveryVM(new SerializableRunnable("testDACKLoadedMessage: confirm standard case") {
          @Override
          public void run() {
            Region rgn1 = getCache().getRegion(rgnName);
            assertEquals("val3", rgn1.getEntry("key3").getValue());
            assertEquals("val4", rgn1.getEntry("key4").getValue());
          }
        });

  }

  @Override
  public Properties getDistributedSystemProperties() {
    Properties p = super.getDistributedSystemProperties();
    p.put(LOG_LEVEL, LogWriterUtils.getDUnitLogLevel());
    p.put(ENABLE_NETWORK_PARTITION_DETECTION, "false");
    return p;
  }

  @Test
  public void testHighAvailabilityFeatures() throws Exception {
    IgnoredException.addIgnoredException("DistributedSystemDisconnectedException");
    final String rgnName = getUniqueName();
    AttributesFactory factory = new AttributesFactory();
    factory.setScope(Scope.DISTRIBUTED_ACK);
    factory.setEarlyAck(false);
    Region rgn = getCache().createRegion(rgnName, factory.create());
    Invoke.invokeInEveryVM(
        new SerializableRunnable("testHighAvailabilityFeatures: intial region configuration") {
          @Override
          public void run() {
            try {
              AttributesFactory factory2 = new AttributesFactory();
              factory2.setScope(Scope.DISTRIBUTED_ACK);
              factory2.setEarlyAck(false);
              factory2.setDataPolicy(DataPolicy.REPLICATE);
              getCache().createRegion(rgnName, factory2.create());
            } catch (CacheException e) {
              Assert.fail("While creating region", e);
            }
          }
        });

    // create entries
    rgn.put("key0", "val0_0");
    rgn.put("key1", "val1_0");

    Host host = Host.getHost(0);
    // This test assumes that there are at least three VMs; the origin and two recipients
    assertTrue(host.getVMCount() >= 3);
    final VM originVM = host.getVM(0);

    // Test that there is no commit after a partial commit message
    // send (only sent to a minority of the recipients)
    originVM.invoke(new SerializableRunnable("Flakey DuringIndividualSend Transaction") {
      @Override
      public void run() {
        final Region rgn1 = getCache().getRegion(rgnName);
        assertNotNull(rgn1);
        try {
          final CacheTransactionManager txMgr2 = getCache().getCacheTransactionManager();
          final CacheTransactionManager txMgrImpl = txMgr2;

          txMgr2.begin();
          // 1. setup an internal callback on originVM that will call
          // disconnectFromDS() on the 2nd duringIndividualSend
          // call.
          ((TXStateProxyImpl) ((TXManagerImpl) txMgrImpl).getTXState()).forceLocalBootstrap();
          TXState txState = (TXState) ((TXStateProxyImpl) ((TXManagerImpl) txMgrImpl).getTXState())
              .getRealDeal(null, null);
          txState.setDuringIndividualSend(new Runnable() {
            private int numCalled = 0;

            @Override
            public synchronized void run() {
              ++numCalled;
              rgn1.getCache().getLogger()
                  .info("setDuringIndividualSend Runnable called " + numCalled + " times");
              if (numCalled > 1) {
                MembershipManagerHelper.crashDistributedSystem(getSystem());
              }
            }
          });
          rgn1.put("key0", "val0_1");
          rgn1.put("key1", "val1_1");
          // 2. commit a transaction in originVM, it will disconnect from the DS
          txMgr2.commit();
        } catch (VirtualMachineError e) {
          SystemFailure.initiateFailure(e);
          throw e;
        } catch (Throwable e) {
          rgn1.getCache().getLogger().warning("Ignoring Exception", e);
        } finally {
          // Allow this VM to re-connect to the DS upon getCache() call
          closeCache();
        }
      }
    });
    // 3. verify on all VMs that the transaction was not committed
    final SerializableRunnable noChangeValidator =
        new SerializableRunnable("testHighAvailabilityFeatures: validate no change in Region") {
          @Override
          public void run() {
            Region rgn1 = getCache().getRegion(rgnName);
            if (rgn1 == null) {
              // Expect a null region from originVM
              try {
                AttributesFactory factory2 = new AttributesFactory();
                factory2.setScope(Scope.DISTRIBUTED_ACK);
                factory2.setEarlyAck(false);
                factory2.setDataPolicy(DataPolicy.REPLICATE);
                rgn1 = getCache().createRegion(rgnName, factory2.create());
              } catch (CacheException e) {
                Assert.fail("While creating region", e);
              }
            }
            Region.Entry re = rgn1.getEntry("key0");
            assertNotNull(re);
            assertEquals("val0_0", re.getValue());
            re = rgn1.getEntry("key1");
            assertNotNull(re);
            assertEquals("val1_0", re.getValue());
          }
        };
    Invoke.invokeInEveryVM(noChangeValidator);

    // Test that there is no commit after sending to all recipients
    // but prior to sending the "commit process" message
    originVM.invoke(new SerializableRunnable("Flakey AfterIndividualSend Transaction") {
      @Override
      public void run() {
        final Region rgn1 = getCache().getRegion(rgnName);
        assertNotNull(rgn1);
        try {
          final CacheTransactionManager txMgr2 = getCache().getCacheTransactionManager();
          final CacheTransactionManager txMgrImpl = txMgr2;

          txMgr2.begin();
          // 1. setup an internal callback on originVM that will call
          // disconnectFromDS() on AfterIndividualSend
          ((TXStateProxyImpl) ((TXManagerImpl) txMgrImpl).getTXState()).forceLocalBootstrap();
          TXState txState = (TXState) ((TXStateProxyImpl) ((TXManagerImpl) txMgrImpl).getTXState())
              .getRealDeal(null, null);
          txState.setAfterIndividualSend(new Runnable() {
            @Override
            public synchronized void run() {
              MembershipManagerHelper.crashDistributedSystem(getSystem());
            }
          });
          rgn1.put("key0", "val0_2");
          rgn1.put("key1", "val1_2");
          // 2. commit a transaction in originVM, it will disconnect from the DS
          txMgr2.commit();
        } catch (VirtualMachineError e) {
          SystemFailure.initiateFailure(e);
          throw e;
        } catch (Throwable e) {
          rgn1.getCache().getLogger().warning("Ignoring Exception", e);
        } finally {
          // Allow this VM to re-connect to the DS upon getCache() call
          closeCache();
        }
      }
    });
    // 3. verify on all VMs, including the origin, that the transaction was not committed
    Invoke.invokeInEveryVM(noChangeValidator);

    // Test commit success upon a single commit process message received.
    originVM.invoke(new SerializableRunnable("Flakey DuringIndividualCommitProcess Transaction") {
      @Override
      public void run() {
        final Region rgn1 = getCache().getRegion(rgnName);
        assertNotNull(rgn1);
        try {
          final CacheTransactionManager txMgr2 = getCache().getCacheTransactionManager();
          final CacheTransactionManager txMgrImpl = txMgr2;

          txMgr2.begin();

          ((TXStateProxyImpl) ((TXManagerImpl) txMgrImpl).getTXState()).forceLocalBootstrap();
          TXState txState = (TXState) ((TXStateProxyImpl) ((TXManagerImpl) txMgrImpl).getTXState())
              .getRealDeal(null, null);
          // 1. setup an internal callback on originVM that will call
          // disconnectFromDS() on the 2nd internalDuringIndividualCommitProcess
          // call.
          txState.setDuringIndividualCommitProcess(new Runnable() {
            private int numCalled = 0;

            @Override
            public synchronized void run() {
              ++numCalled;
              rgn1.getCache().getLogger()
                  .info("setDuringIndividualCommitProcess Runnable called " + numCalled + " times");
              if (numCalled > 1) {
                MembershipManagerHelper.crashDistributedSystem(getSystem());
              }
            }
          });
          rgn1.put("key0", "val0_3");
          rgn1.put("key1", "val1_3");
          // 2. commit a transaction in originVM, it will disconnect from the DS
          txMgr2.commit();
        } catch (VirtualMachineError e) {
          SystemFailure.initiateFailure(e);
          throw e;
        } catch (Throwable e) {
          rgn1.getCache().getLogger().warning("Ignoring Exception", e);
        } finally {
          // Allow this VM to re-connect to the DS upon getCache() call
          closeCache();
        }
      }
    });
    // 3. verify on all VMs that the transaction was committed (including the orgin, due to GII)
    SerializableRunnable nonSoloChangeValidator1 = new SerializableRunnable(
        "testHighAvailabilityFeatures: validate v1 non-solo Region changes") {
      @Override
      public void run() {
        Region rgn1 = getCache().getRegion(rgnName);
        if (rgn1 == null) {
          // Expect a null region from originVM
          try {
            AttributesFactory factory2 = new AttributesFactory();
            factory2.setScope(Scope.DISTRIBUTED_ACK);
            factory2.setEarlyAck(false);
            factory2.setDataPolicy(DataPolicy.REPLICATE);
            rgn1 = getCache().createRegion(rgnName, factory2.create());
          } catch (CacheException e) {
            Assert.fail("While creating region", e);
          }
        }
        long giveUp = System.currentTimeMillis() + 10000;
        while (giveUp > System.currentTimeMillis()) {
          try {
            Region.Entry re = rgn1.getEntry("key0");
            assertNotNull(re);
            assertEquals("val0_3", re.getValue());
            re = rgn1.getEntry("key1");
            assertNotNull(re);
            assertEquals("val1_3", re.getValue());
            break;
          } catch (AssertionError e) {
            if (giveUp > System.currentTimeMillis()) {
              throw e;
            }
          }
        }
      }
    };
    Invoke.invokeInEveryVM(nonSoloChangeValidator1);

    // Verify successful solo region commit after duringIndividualSend
    // (same as afterIndividualSend).
    // Create a region that only exists on the origin and another VM
    final String soloRegionName = getUniqueName() + "_solo";
    SerializableRunnable createSoloRegion =
        new SerializableRunnable("testHighAvailabilityFeatures: solo region configuration") {
          @Override
          public void run() {
            try {
              AttributesFactory factory2 = new AttributesFactory();
              factory2.setScope(Scope.DISTRIBUTED_ACK);
              factory2.setEarlyAck(false);
              factory2.setDataPolicy(DataPolicy.REPLICATE);
              Region rgn1 = getCache().createRegion(soloRegionName, factory2.create());
              rgn1.put("soloKey0", "soloVal0_0");
              rgn1.put("soloKey1", "soloVal1_0");
            } catch (CacheException e) {
              Assert.fail("While creating region", e);
            }
          }
        };
    final VM soloRegionVM = host.getVM(1);
    originVM.invoke(createSoloRegion);
    soloRegionVM.invoke(createSoloRegion);
    originVM
        .invoke(new SerializableRunnable("Flakey solo region DuringIndividualSend Transaction") {
          @Override
          public void run() {
            final Region soloRgn = getCache().getRegion(soloRegionName);
            assertNotNull(soloRgn);
            try {
              final CacheTransactionManager txMgr2 = getCache().getCacheTransactionManager();
              final CacheTransactionManager txMgrImpl = txMgr2;

              txMgr2.begin();
              // 1. setup an internal callback on originVM that will call
              // disconnectFromDS() on the 2nd duringIndividualSend
              // call.
              ((TXStateProxyImpl) ((TXManagerImpl) txMgrImpl).getTXState()).forceLocalBootstrap();
              TXState txState =
                  (TXState) ((TXStateProxyImpl) ((TXManagerImpl) txMgrImpl).getTXState())
                      .getRealDeal(null, null);
              txState.setDuringIndividualSend(new Runnable() {
                private int numCalled = 0;

                @Override
                public synchronized void run() {
                  ++numCalled;
                  soloRgn.getCache().getLogger()
                      .info("setDuringIndividualSend Runnable called " + numCalled + " times");
                  if (numCalled > 1) {
                    MembershipManagerHelper.crashDistributedSystem(getSystem());
                  }
                }
              });
              soloRgn.put("soloKey0", "soloVal0_1");
              soloRgn.put("soloKey1", "soloVal1_1");
              // 2. commit a transaction in originVM, it will disconnect from the DS
              txMgr2.commit();
            } catch (VirtualMachineError e) {
              SystemFailure.initiateFailure(e);
              throw e;
            } catch (Throwable e) {
              soloRgn.getCache().getLogger().warning("Ignoring Exception", e);
            } finally {
              // Allow this VM to re-connect to the DS upon getCache() call
              closeCache();
            }
          }
        });
    // 3. verify on the soloRegionVM that the transaction was committed
    final SerializableRunnable soloRegionCommitValidator1 = new SerializableRunnable(
        "testHighAvailabilityFeatures: validate successful v1 commit in solo Region") {
