/*
 * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
 * agreements. See the NOTICE file distributed with this work for additional information regarding
 * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance with the License. You may obtain a
 * copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */
package org.apache.geode.internal.cache;

import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static org.apache.geode.distributed.ConfigurationProperties.LOCATORS;
import static org.apache.geode.distributed.ConfigurationProperties.LOG_LEVEL;
import static org.apache.geode.distributed.ConfigurationProperties.MCAST_PORT;
import static org.apache.geode.distributed.ConfigurationProperties.OFF_HEAP_MEMORY_SIZE;
import static org.apache.geode.internal.AvailablePortHelper.getRandomAvailableTCPPort;
import static org.apache.geode.test.awaitility.GeodeAwaitility.await;
import static org.apache.geode.test.dunit.LogWriterUtils.getDUnitLogLevel;
import static org.apache.geode.test.dunit.LogWriterUtils.getLogWriter;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNotSame;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicInteger;

import javax.naming.Context;

import jakarta.transaction.RollbackException;
import jakarta.transaction.Synchronization;
import jakarta.transaction.UserTransaction;
import org.junit.Ignore;
import org.junit.Test;

import org.apache.geode.cache.AttributesFactory;
import org.apache.geode.cache.Cache;
import org.apache.geode.cache.CacheEvent;
import org.apache.geode.cache.CacheFactory;
import org.apache.geode.cache.CacheListener;
import org.apache.geode.cache.CacheLoader;
import org.apache.geode.cache.CacheLoaderException;
import org.apache.geode.cache.CacheTransactionManager;
import org.apache.geode.cache.CacheWriterException;
import org.apache.geode.cache.CommitConflictException;
import org.apache.geode.cache.DataPolicy;
import org.apache.geode.cache.EntryEvent;
import org.apache.geode.cache.InterestPolicy;
import org.apache.geode.cache.LoaderHelper;
import org.apache.geode.cache.PartitionAttributesFactory;
import org.apache.geode.cache.Region;
import org.apache.geode.cache.Region.Entry;
import org.apache.geode.cache.RegionFactory;
import org.apache.geode.cache.RegionShortcut;
import org.apache.geode.cache.Scope;
import org.apache.geode.cache.SubscriptionAttributes;
import org.apache.geode.cache.TransactionDataNodeHasDepartedException;
import org.apache.geode.cache.TransactionDataNotColocatedException;
import org.apache.geode.cache.TransactionEvent;
import org.apache.geode.cache.TransactionException;
import org.apache.geode.cache.TransactionId;
import org.apache.geode.cache.TransactionInDoubtException;
import org.apache.geode.cache.TransactionWriter;
import org.apache.geode.cache.TransactionWriterException;
import org.apache.geode.cache.UnsupportedOperationInTransactionException;
import org.apache.geode.cache.client.ClientCache;
import org.apache.geode.cache.client.ClientCacheFactory;
import org.apache.geode.cache.client.ClientRegionFactory;
import org.apache.geode.cache.client.ClientRegionShortcut;
import org.apache.geode.cache.client.PoolFactory;
import org.apache.geode.cache.client.PoolManager;
import org.apache.geode.cache.execute.Execution;
import org.apache.geode.cache.execute.FunctionAdapter;
import org.apache.geode.cache.execute.FunctionContext;
import org.apache.geode.cache.execute.FunctionException;
import org.apache.geode.cache.execute.FunctionService;
import org.apache.geode.cache.execute.RegionFunctionContext;
import org.apache.geode.cache.server.CacheServer;
import org.apache.geode.cache.util.CacheListenerAdapter;
import org.apache.geode.cache.util.CacheWriterAdapter;
import org.apache.geode.cache.util.TransactionListenerAdapter;
import org.apache.geode.distributed.ConfigurationProperties;
import org.apache.geode.distributed.DistributedMember;
import org.apache.geode.distributed.internal.InternalDistributedSystem;
import org.apache.geode.distributed.internal.membership.InternalDistributedMember;
import org.apache.geode.internal.cache.execute.data.CustId;
import org.apache.geode.internal.cache.execute.data.Customer;
import org.apache.geode.internal.cache.execute.data.Order;
import org.apache.geode.internal.cache.execute.data.OrderId;
import org.apache.geode.internal.cache.tx.ClientTXStateStub;
import org.apache.geode.internal.jta.SyncImpl;
import org.apache.geode.internal.jta.TransactionImpl;
import org.apache.geode.internal.jta.TransactionManagerImpl;
import org.apache.geode.internal.jta.UserTransactionImpl;
import org.apache.geode.test.awaitility.GeodeAwaitility;
import org.apache.geode.test.dunit.Assert;
import org.apache.geode.test.dunit.DistributedTestUtils;
import org.apache.geode.test.dunit.Host;
import org.apache.geode.test.dunit.IgnoredException;
import org.apache.geode.test.dunit.SerializableCallable;
import org.apache.geode.test.dunit.SerializableRunnable;
import org.apache.geode.test.dunit.VM;
import org.apache.geode.test.dunit.cache.internal.JUnit4CacheTestCase;
import org.apache.geode.test.dunit.internal.JUnit4DistributedTestCase;
import org.apache.geode.util.internal.GeodeGlossary;

/**
 * Tests the basic client-server transaction functionality
 */

public class ClientServerTransactionDUnitTest extends RemoteTransactionDUnitTest {

  protected static final int MAX_ENTRIES = 10;

  private enum forop {
    CREATE, UPDATE, DESTROY
  }

  protected static final String OTHER_REGION = "OtherRegion";

  public ClientServerTransactionDUnitTest() {
    super();
  }

  @Override
  public final void postSetUp() throws Exception {
    IgnoredException.addIgnoredException("java.net.SocketException");
    postSetUpClientServerTransactionDUnitTest();
  }

  protected void postSetUpClientServerTransactionDUnitTest() throws Exception {}


  @Override
  public Properties getDistributedSystemProperties() {
    Properties result = super.getDistributedSystemProperties();
    result.put(ConfigurationProperties.SERIALIZABLE_OBJECT_FILTER,
        result.get(ConfigurationProperties.SERIALIZABLE_OBJECT_FILTER)
            + ";org.apache.geode.internal.cache.ClientServerTransactionDUnitTest*"
            + ";org.apache.geode.test.dunit.**" + ";org.apache.geode.test.junit.**");
    return result;
  }


  private Integer createRegionsAndStartServerWithTimeout(VM vm, boolean accessor,
      int txTimeoutSecs) {
    return createRegionOnServerWithTimeout(vm, true, accessor, txTimeoutSecs);
  }

  private Integer createRegionOnServerWithTimeout(VM vm, final boolean startServer,
      final boolean accessor, final int txTimeoutSecs) {
    return createRegionOnServerWithTimeout(vm, startServer, accessor, 0, txTimeoutSecs);
  }

  private Integer createRegionsAndStartServer(VM vm, boolean accessor) {
    return createRegionOnServer(vm, true, accessor);
  }

  private void createRegionOnServer(VM vm) {
    createRegionOnServer(vm, false, false);
  }

  private Integer createRegionOnServer(VM vm, final boolean startServer, final boolean accessor) {
    return createRegionOnServer(vm, startServer, accessor, 0);
  }

  private Integer createRegionOnServer(VM vm, final boolean startServer, final boolean accessor,
      final int redundantCopies) {
    return createRegionOnServerWithTimeout(vm, startServer, accessor, redundantCopies, 10);
  }

  private Integer createRegionOnServerWithTimeout(VM vm, final boolean startServer,
      final boolean accessor, final int redundantCopies, final int txTimeoutSecs) {
    return (Integer) vm.invoke(new SerializableCallable<Object>() {
      @Override
      public Object call() throws Exception {
        createRegion(accessor, redundantCopies, null);
        TXManagerImpl txMgr = (TXManagerImpl) getCache().getCacheTransactionManager();
        txMgr.setTransactionTimeToLiveForTest(txTimeoutSecs);
        if (startServer) {
          int port = getRandomAvailableTCPPort();
          CacheServer s = getCache().addCacheServer();
          s.setPort(port);
          s.start();
          return port;
        }
        return 0;
      }
    });
  }

  private Integer createRegionOnDisconnectedServer(VM vm, final boolean startServer) {
    return (Integer) vm.invoke(new SerializableCallable<Object>() {
      @Override
      public Object call() throws Exception {
        AttributesFactory af = new AttributesFactory();
        af.setScope(Scope.DISTRIBUTED_ACK);
        af.setDataPolicy(DataPolicy.REPLICATE);
        Properties props = getDistributedSystemProperties();
        props.put(MCAST_PORT, "0");
        props.remove(LOCATORS);
        InternalDistributedSystem system = getSystem(props);
        Cache cache = CacheFactory.create(system);
        cache.createRegion(OTHER_REGION, af.create());
        TXManagerImpl txMgr = (TXManagerImpl) cache.getCacheTransactionManager();
        txMgr.setTransactionTimeToLiveForTest(10);
        if (startServer) {
          int port = getRandomAvailableTCPPort();
          CacheServer s = cache.addCacheServer();
          s.setPort(port);
          s.start();
          return port;
        }
        return 0;
      }
    });
  }


  // private void createClientRegionWithRI(VM vm, final int port, final boolean isEmpty) {
  // createClientRegion(vm, port, isEmpty, true);
  // }

  private void createClientRegion(VM vm, final int port, final boolean isEmpty) {
    createClientRegion2(vm, port, isEmpty, false);
  }

  private void createClientRegionAndPopulateData(VM vm, final int port, final boolean isEmpty) {
    createClientRegion(vm, port, isEmpty);
    vm.invoke(new SerializableCallable<Object>() {
      @Override
      public Object call() {
        populateData();
        return null;
      }
    });
  }

  private void createClientRegion2(VM vm, final int port, final boolean isEmpty, final boolean ri) {
    vm.invoke(new SerializableCallable<Object>() {
      @Override
      public Object call() {
        ClientCacheFactory ccf = new ClientCacheFactory();
        setCCF(port, ccf);
        // these settings were used to manually check that tx operation stats were being updated
        // ccf.set(STATISTIC_SAMPLING_ENABLED, "true");
        // ccf.set(STATISTIC_ARCHIVE_FILE, "clientStats.gfs");
        ClientCache cCache = getClientCache(ccf);
        ClientRegionFactory<Integer, String> crf = cCache.createClientRegionFactory(
            isEmpty ? ClientRegionShortcut.PROXY : ClientRegionShortcut.CACHING_PROXY);
        crf.setConcurrencyChecksEnabled(getConcurrencyChecksEnabled());
        Region<Integer, String> r = crf.create(D_REFERENCE);
        Region<Integer, String> customer = crf.create(CUSTOMER);
        Region<Integer, String> order = crf.create(ORDER);
        if (ri) {
          r.registerInterestRegex(".*");
          customer.registerInterestRegex(".*");
          order.registerInterestRegex(".*");
        }
        return null;
      }
    });
  }

  /*
   * (non-Javadoc)
   *
   * @see org.apache.geode.internal.cache.RemoteTransactionDUnitTest#getVMForTransactions(dunit.VM,
   * dunit.VM)
   */
  @Override
  public VM getVMForTransactions(VM accessor, VM datastore) {
    // create a cache server in the accessor VM and then create and return
    // a client VM
    final int serverPort =
        (Integer) accessor.invoke(new SerializableCallable("create cache server") {
          @Override
          public Object call() throws Exception {
            TXManagerImpl txMgr = (TXManagerImpl) getCache().getCacheTransactionManager();
            txMgr.setTransactionTimeToLiveForTest(10);
            int port = getRandomAvailableTCPPort();
            CacheServer s = getCache().addCacheServer();
            s.setPort(port);
            s.start();
            return port;
          }
        });
    VM clientVM = Host.getHost(0).getVM(2); // superclass always uses 0 and 1
    createClientRegion(clientVM, serverPort, false, false);
    return clientVM;
  }

  private void configureOffheapSystemProperty() {
    Properties p = getDistributedSystemProperties();
    // p.setProperty(LOG_LEVEL, "finer");
    p.setProperty(OFF_HEAP_MEMORY_SIZE, "1m");
    getSystem(p);
  }

  private void createSubscriptionRegion(boolean isOffHeap, String regionName, int copies,
      int totalBuckets) {
    PartitionAttributesFactory paf = new PartitionAttributesFactory();
    paf.setRedundantCopies(copies).setTotalNumBuckets(totalBuckets);
    AttributesFactory attr = new AttributesFactory();
    attr.setPartitionAttributes(paf.create());
    attr.setConcurrencyChecksEnabled(true);
    if (isOffHeap) {
      attr.setOffHeap(isOffHeap);
    }
    Region offheapRegion = getCache().createRegion(regionName, attr.create());
    assertNotNull(offheapRegion);
  }

  private void createClient(int port, String regionName) throws Exception {
    System.setProperty(GeodeGlossary.GEMFIRE_PREFIX + "bridge.disableShufflingOfEndpoints",
        "true");
    ClientCacheFactory ccf = new ClientCacheFactory();
    ccf.addPoolServer("localhost"/* getServerHostName(Host.getHost(0)) */, port);
    ccf.setPoolMinConnections(0);
    ccf.setPoolSubscriptionEnabled(true);
    ccf.setPoolSubscriptionRedundancy(0);
    ccf.set(LOG_LEVEL, getDUnitLogLevel());
    ClientCache cCache = getClientCache(ccf);
    Region r = cCache.createClientRegionFactory(ClientRegionShortcut.CACHING_PROXY)
        .addCacheListener(new ClientCacheListener()).create(regionName);
    r.registerInterestRegex(".*");
  }

  class ClientCacheListener extends CacheListenerAdapter {
    private int eventCount;

    @Override
    public void afterCreate(EntryEvent event) {
      onEvent(event);
    }

    @Override
    public void afterUpdate(EntryEvent event) {
      onEvent(event);
    }

    @Override
    public void afterDestroy(EntryEvent event) {
      onEvent(event);
    }

    private void onEvent(EntryEvent event) {
      eventCount++;
    }

    public int getEventCount() {
      return eventCount;
    }
  }

  private int getClientCacheListnerEventCount(String regionName) {
    Region r = getCache().getRegion(regionName);
    CacheListener<?, ?>[] listeners = r.getAttributes().getCacheListeners();
    for (CacheListener<?, ?> listener : listeners) {
      if (listener instanceof ClientCacheListener) {
        return ((ClientCacheListener) listener).getEventCount();
      }
    }
    return 0;
  }


  @Test
  public void testTwoPoolsNotAllowed() {
    Host host = Host.getHost(0);
    VM datastore1 = host.getVM(0);
    VM datastore2 = host.getVM(1);
    final boolean cachingProxy = false;

    disconnectAllFromDS(); // some other VMs seem to be hanging around and have the region this
                           // tests uses

    final int port1 = createRegionsAndStartServer(datastore1, false);
    System.setProperty(GeodeGlossary.GEMFIRE_PREFIX + "bridge.disableShufflingOfEndpoints",
        "true");
    ClientCacheFactory ccf = new ClientCacheFactory();
    setCCF(port1, ccf);

    ClientCache cCache = getClientCache(ccf);

    ClientRegionFactory<CustId, Customer> custrf = cCache.createClientRegionFactory(
        cachingProxy ? ClientRegionShortcut.CACHING_PROXY : ClientRegionShortcut.PROXY);
    ClientRegionFactory<Integer, String> refrf = cCache.createClientRegionFactory(
        cachingProxy ? ClientRegionShortcut.CACHING_PROXY : ClientRegionShortcut.PROXY);
    Region<Integer, String> r = refrf.create(D_REFERENCE);
    Region<CustId, Customer> pr = custrf.create(CUSTOMER);

    // set up a second pool for the other distributed system's region
    final int port2 = createRegionOnDisconnectedServer(datastore2, true);
    PoolFactory pf = PoolManager.createFactory();
    pf.addServer("localhost"/* getServerHostName(Host.getHost(0)) */, port2);
    pf.create("otherServer");

    ClientRegionFactory otherrf = cCache.createClientRegionFactory(
        cachingProxy ? ClientRegionShortcut.CACHING_PROXY : ClientRegionShortcut.PROXY);
    otherrf.setPoolName("otherServer");
    Region<Object, Object> otherRegion = otherrf.create(OTHER_REGION);

    TXManagerImpl mgr = getGemfireCache().getTxManager();
    mgr.begin();
    doTxOps(r, pr);
    boolean exceptionThrown = false;
    try {
      otherRegion.put("tx", "not allowed");
    } catch (TransactionException expected) {
      exceptionThrown = true;
    }

    cCache.close();
    datastore1.invoke(JUnit4CacheTestCase::closeCache);
    datastore2.invoke(JUnit4CacheTestCase::closeCache);

    if (!exceptionThrown) {
      fail("expected TransactionException to be thrown since two pools were used");
    }
  }

  @Test
  public void testCleanupAfterClientFailure() {
    Host host = Host.getHost(0);
    VM accessor = host.getVM(0);
    VM datastore = host.getVM(1);
    final boolean cachingProxy = false;

    disconnectAllFromDS(); // some other VMs seem to be hanging around and have the region this
                           // tests uses

    final int port1 = createRegionsAndStartServerWithTimeout(accessor, true, 5);
    createRegionOnServerWithTimeout(datastore, false, false, 5);

    System.setProperty(GeodeGlossary.GEMFIRE_PREFIX + "bridge.disableShufflingOfEndpoints",
        "true");
    ClientCacheFactory ccf = new ClientCacheFactory();
    setCCF(port1, ccf);
    ClientCache cCache = getClientCache(ccf);
    ClientRegionFactory<CustId, Customer> custrf = cCache.createClientRegionFactory(
        cachingProxy ? ClientRegionShortcut.CACHING_PROXY : ClientRegionShortcut.PROXY);
    ClientRegionFactory<Integer, String> refrf = cCache.createClientRegionFactory(
        cachingProxy ? ClientRegionShortcut.CACHING_PROXY : ClientRegionShortcut.PROXY);
    Region<Integer, String> r = refrf.create(D_REFERENCE);
    Region<CustId, Customer> pr = custrf.create(CUSTOMER);

    TXManagerImpl mgr = getGemfireCache().getTxManager();
    mgr.begin();
    doTxOps(r, pr);

    final DistributedMember myId = cCache.getDistributedSystem().getDistributedMember();

    SerializableCallable verifyExists =
        new SerializableCallable("verify txstate for client exists") {
          @Override
          public Object call() {
            TXManagerImpl txmgr = getGemfireCache().getTxManager();
            Set states = txmgr.getTransactionsForClient((InternalDistributedMember) myId);
            assertEquals(1, states.size()); // only one in-progress transaction
            return null;
          }
        };

    accessor.invoke(verifyExists);
    datastore.invoke(verifyExists);

    cCache.close();

    SerializableCallable verifyExpired = new SerializableCallable("verify txstate is expired") {
      @Override
      public Object call() {
        final TXManagerImpl txmgr = getGemfireCache().getTxManager();
        try {
          GeodeAwaitility.await().untilAsserted(() -> {
            Set states = txmgr.getTransactionsForClient((InternalDistributedMember) myId);
            getLogWriter()
                .info("found " + states.size() + " tx states for " + myId);
            assertThat(states).as("Waiting for transaction state to expire").isEmpty();
          });
          return null;
        } finally {
          getGemfireCache().getDistributedSystem().disconnect();
        }
      }
    };
    try {
      accessor.invoke(verifyExpired);
      datastore.invoke(verifyExpired);
    } finally {
      cCache.close();
    }
  }

  @Test
  public void testCleanupAfterClientAndProxyFailure() {
    Host host = Host.getHost(0);
    VM accessor = host.getVM(0);
    VM datastore = host.getVM(1);
    final boolean cachingProxy = false;

    disconnectAllFromDS(); // some other VMs seem to be hanging around and have the region this
                           // tests uses

    final int port1 = createRegionsAndStartServerWithTimeout(accessor, true, 5);
    createRegionOnServerWithTimeout(datastore, false, false, 5);

    System.setProperty(GeodeGlossary.GEMFIRE_PREFIX + "bridge.disableShufflingOfEndpoints",
        "true");
    ClientCacheFactory ccf = new ClientCacheFactory();
    setCCF(port1, ccf);
    ClientCache cCache = getClientCache(ccf);
    ClientRegionFactory<CustId, Customer> custrf = cCache.createClientRegionFactory(
        cachingProxy ? ClientRegionShortcut.CACHING_PROXY : ClientRegionShortcut.PROXY);
    ClientRegionFactory<Integer, String> refrf = cCache.createClientRegionFactory(
        cachingProxy ? ClientRegionShortcut.CACHING_PROXY : ClientRegionShortcut.PROXY);
    Region<Integer, String> r = refrf.create(D_REFERENCE);
    Region<CustId, Customer> pr = custrf.create(CUSTOMER);

    TXManagerImpl mgr = getGemfireCache().getTxManager();
    mgr.begin();
    doTxOps(r, pr);

    final DistributedMember myId = cCache.getDistributedSystem().getDistributedMember();

    SerializableCallable verifyExists =
        new SerializableCallable("verify txstate for client exists") {
          @Override
          public Object call() {
            TXManagerImpl txmgr = getGemfireCache().getTxManager();
            Set states = txmgr.getTransactionsForClient((InternalDistributedMember) myId);
            assertEquals(1, states.size()); // only one in-progress transaction
            return null;
          }
        };

    accessor.invoke(verifyExists);
    datastore.invoke(verifyExists);

    accessor.invoke(JUnit4CacheTestCase::closeCache);
    accessor.invoke(JUnit4DistributedTestCase::disconnectFromDS);

    SerializableCallable verifyExpired = new SerializableCallable("verify txstate is expired") {
      @Override
      public Object call() {
        final TXManagerImpl txmgr = getGemfireCache().getTxManager();
        return verifyTXStateExpired(myId, txmgr);
      }
    };
    try {
      datastore.invoke(verifyExpired);
    } finally {
      cCache.close();
    }
  }

  void doTxOps(Region<Integer, String> r, Region<CustId, Customer> pr) {
    for (int i = 0; i < 5; i++) {
      CustId custId = new CustId(i);
      Customer cust = new Customer("name" + i, "address" + i);
      getGemfireCache().getLogger().info("putting:" + custId);
      pr.put(custId, cust);
      r.put(i, "value" + i);
    }
  }

  public static DistributedMember getVMDistributedMember() {
    return InternalDistributedSystem.getAnyInstance().getDistributedMember();
  }

  @SuppressWarnings("unchecked")
  @Test
  public void testFailoverAfterProxyFailure() throws InterruptedException {
    Host host = Host.getHost(0);
    VM accessor = host.getVM(0);
    VM datastore = host.getVM(1);
    VM accessor2 = host.getVM(2);
    final boolean cachingProxy = false;

    disconnectAllFromDS(); // some other VMs seem to be hanging around and have the region this
                           // tests uses

    int[] ports = new int[2];
    ports[0] = createRegionsAndStartServerWithTimeout(accessor, true, 5);
    ports[1] = createRegionsAndStartServerWithTimeout(accessor2, true, 5);
    createRegionOnServerWithTimeout(datastore, false, false, 5);

    System.setProperty(GeodeGlossary.GEMFIRE_PREFIX + "bridge.disableShufflingOfEndpoints",
        "true");
    ClientCacheFactory ccf = new ClientCacheFactory();
    setCCF(ports, ccf);
    ClientCache cCache = getClientCache(ccf);
    ClientRegionFactory<CustId, Customer> custrf = cCache.createClientRegionFactory(
        cachingProxy ? ClientRegionShortcut.CACHING_PROXY : ClientRegionShortcut.PROXY);
    ClientRegionFactory<Integer, String> refrf = cCache.createClientRegionFactory(
        cachingProxy ? ClientRegionShortcut.CACHING_PROXY : ClientRegionShortcut.PROXY);
    Region<Integer, String> r = refrf.create(D_REFERENCE);
    Region<CustId, Customer> pr = custrf.create(CUSTOMER);

    TXManagerImpl mgr = getGemfireCache().getTxManager();
    mgr.begin();
    doTxOps(r, pr);

    final DistributedMember myId = cCache.getDistributedSystem().getDistributedMember();
    final DistributedMember accessorId = accessor
        .invoke(ClientServerTransactionDUnitTest::getVMDistributedMember);
    final DistributedMember accessor2Id = accessor2
        .invoke(ClientServerTransactionDUnitTest::getVMDistributedMember);

    SerializableCallable verifyExists =
        new SerializableCallable("verify txstate for client exists") {
          @Override
          public Object call() {
            TXManagerImpl txmgr = getGemfireCache().getTxManager();
            Set<TXId> states = txmgr.getTransactionsForClient((InternalDistributedMember) myId);
            assertEquals(1, states.size()); // only one in-progress transaction
            return null;
          }
        };

    datastore.invoke(verifyExists);

    SerializableCallable getProxyServer = new SerializableCallable("get proxy server") {
      @Override
      public Object call() {
        final TXManagerImpl txmgr = getGemfireCache().getTxManager();
        DistributedMember proxyServer = null;
        TXStateProxyImpl tx = null;
        Set<TXStateProxy> states =
            txmgr.getTransactionStatesForClient((InternalDistributedMember) myId);
        assertEquals(1, states.size());
        Iterator<TXStateProxy> iterator = states.iterator();
        if (iterator.hasNext()) {
          tx = (TXStateProxyImpl) iterator.next();
          assertTrue(tx.isRealDealLocal());
          proxyServer = ((TXState) tx.realDeal).getProxyServer();
        }
        return proxyServer;
      }
    };

    final DistributedMember proxy = (DistributedMember) datastore.invoke(getProxyServer);

    if (proxy.equals(accessorId)) {
      accessor.invoke(JUnit4CacheTestCase::closeCache);
      accessor.invoke(JUnit4DistributedTestCase::disconnectFromDS);
    } else {
      assertTrue(proxy.equals(accessor2Id));
      accessor2.invoke(JUnit4CacheTestCase::closeCache);
      accessor2.invoke(JUnit4DistributedTestCase::disconnectFromDS);
    }

    doTxOps(r, pr);

    SerializableCallable verifyProxyServerChanged =
        new SerializableCallable("verify proxy server is updated") {
          @Override
          public Object call() {
            final TXManagerImpl txmgr = getGemfireCache().getTxManager();
            TXStateProxyImpl tx = null;
            Set<TXStateProxy> states =
                txmgr.getTransactionStatesForClient((InternalDistributedMember) myId);
            assertEquals(1, states.size());
            Iterator<TXStateProxy> iterator = states.iterator();
            if (iterator.hasNext()) {
              tx = (TXStateProxyImpl) iterator.next();
              assertTrue(tx.isRealDealLocal());
            }
            return verifyProxyServerChanged(tx, proxy);
          }
        };
    try {
      datastore.invoke(verifyProxyServerChanged);
    } finally {
      cCache.close();
    }
  }

  void setCCF(final int port1, ClientCacheFactory ccf) {
    ccf.addPoolServer("localhost"/* getServerHostName(Host.getHost(0)) */, port1);
    ccf.setPoolSubscriptionEnabled(false);
    ccf.set(LOG_LEVEL, getDUnitLogLevel());
  }

  void setCCF(final int[] ports, ClientCacheFactory ccf) {
    for (int port : ports) {
      ccf.addPoolServer("localhost", port);
    }
    ccf.setPoolSubscriptionEnabled(false);
    ccf.set(LOG_LEVEL, getDUnitLogLevel());
  }

  @Test
  public void testBasicCommitOnEmpty() {
    doBasicTransaction(false, false, true);
  }

  @Test
  public void testBasicCommitOnEmptyUsingJTA() {
    doBasicTransaction(false, true, true);
  }

  @Test
  public void testBasicCommit() {
    doBasicTransaction(true, false, true);
  }

  @Test
  public void testBasicCommitUsingJTA() {
    doBasicTransaction(true, true, true);
  }

  @Test
  public void testBasicRollbackUsingJTA() {
    doBasicTransaction(true, true, false);
  }

  private void doBasicTransaction(final boolean prePopulateData, final boolean useJTA,
      final boolean isCommit) {
    Host host = Host.getHost(0);
    VM server = host.getVM(0);
    VM client = host.getVM(1);
    int port1 = createRegionsAndStartServer(server, false);
    if (prePopulateData) {
      createClientRegionAndPopulateData(client, port1, false);
    } else {
      createClientRegion(client, port1, false);
    }

    client.invoke(new SerializableCallable<Object>() {
      @Override
      public Object call() throws Exception {
        String suffix = prePopulateData ? "Updated" : "";
        Region<CustId, Customer> r = getGemfireCache().getRegion(D_REFERENCE);
        Region<CustId, Customer> pr = getGemfireCache().getRegion(CUSTOMER);
        int initSize = prePopulateData ? 5 : 0;
        assertEquals(initSize, pr.size());
        assertEquals(initSize, r.size());

        org.apache.geode.test.dunit.LogWriterUtils.getLogWriter()
            .info("Looking up transaction manager");
        TXManagerImpl mgr = (TXManagerImpl) getCache().getCacheTransactionManager();
        Context ctx = getCache().getJNDIContext();
        UserTransaction utx = (UserTransaction) ctx.lookup("java:/UserTransaction");
        org.apache.geode.test.dunit.LogWriterUtils.getLogWriter().info("starting transaction");
        if (useJTA) {
          utx.begin();
        } else {
          mgr.begin();
        }
        org.apache.geode.test.dunit.LogWriterUtils.getLogWriter().info("done starting transaction");
        for (int i = 0; i < MAX_ENTRIES; i++) {
          CustId custId = new CustId(i);
          Customer cust = new Customer("name" + suffix + i, "address" + suffix + i);
          r.put(custId, cust);
          pr.put(custId, cust);
        }
        for (int i = 0; i < MAX_ENTRIES; i++) {
          CustId custId = new CustId(i);
          Customer cust = new Customer("name" + suffix + i, "address" + suffix + i);
          assertEquals(cust, r.get(custId));
          assertEquals(cust, pr.get(custId));
          org.apache.geode.test.dunit.LogWriterUtils.getLogWriter()
              .info("SWAP:get:" + r.get(custId));
          org.apache.geode.test.dunit.LogWriterUtils.getLogWriter()
              .info("SWAP:get:" + pr.get(custId));
        }
        org.apache.geode.test.dunit.LogWriterUtils.getLogWriter().info("suspending transaction");
        if (!useJTA) {
          TXStateProxy tx = mgr.pauseTransaction();
          if (prePopulateData) {
            for (int i = 0; i < 5; i++) {
              CustId custId = new CustId(i);
              Customer cust = new Customer("customer" + i, "address" + i);
              assertEquals(cust, r.get(custId));
              assertEquals(cust, pr.get(custId));
            }
          }
          for (int i = 5; i < MAX_ENTRIES; i++) {
            assertNull(r.get(new CustId(i)));
            assertNull(pr.get(new CustId(i)));
          }
          org.apache.geode.test.dunit.LogWriterUtils.getLogWriter().info("resuming transaction");
          mgr.unpauseTransaction(tx);
        }
        assertEquals("r sized should be " + MAX_ENTRIES + " but it is:" + r.size(), MAX_ENTRIES,
            r.size());
        assertEquals("pr sized should be " + MAX_ENTRIES + " but it is:" + pr.size(), MAX_ENTRIES,
            pr.size());
        org.apache.geode.test.dunit.LogWriterUtils.getLogWriter().info("committing transaction");
        if (isCommit) {
          if (useJTA) {
            utx.commit();
          } else {
            getCache().getCacheTransactionManager().commit();
          }
        } else {
          if (useJTA) {
            utx.rollback();
          } else {
            getCache().getCacheTransactionManager().rollback();
          }
        }
        org.apache.geode.test.dunit.LogWriterUtils.getLogWriter()
            .info("done " + (isCommit ? "committing" : "rollingback") + "transaction");
        int expectedRegionSize = isCommit ? MAX_ENTRIES : 5;

        assertEquals("r sized should be " + expectedRegionSize + " but it is:" + r.size(),
            expectedRegionSize, r.size());
        assertEquals("pr sized should be " + expectedRegionSize + " but it is:" + pr.size(),
            expectedRegionSize, pr.size());

