/*
 * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
 * agreements. See the NOTICE file distributed with this work for additional information regarding
 * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance with the License. You may obtain a
 * copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */
package org.apache.geode.internal.cache;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;

import java.util.Collection;
import java.util.Iterator;
import java.util.Map;
import java.util.Random;
import java.util.concurrent.BrokenBarrierException;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CyclicBarrier;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;

import org.junit.Test;

import org.apache.geode.cache.AttributesFactory;
import org.apache.geode.cache.Region;
import org.apache.geode.cache.Scope;
import org.apache.geode.test.dunit.ThreadUtils;

/**
 * This is a multi threaded tests. This test creates two regions. Region1 which is persistent and
 * Region2 which is not. There will be four sets of threads. One set doing put, second one doing
 * gets ,third one doing destroy(key) and the fourth one doing force rolling.
 *
 * The put are done for Integer Key objects which are random generated and whose values are between
 * 0-9 and the Integer value objects whose value can be between -99 to 99.
 *
 * Since the keys are only 0-9, this will ensure a high level of concurrency on the same thread
 * since there are more than 10 threads acting at the same time.
 *
 * After all the operations are done, the two regions are checked for equality. After that the
 * persistent region is closed and recreated so that it can recover the old values and again the two
 * regions are checked for equality. * This test is run for all modes persist, persist+overflow,
 * overflow only in syn and async mode.
 */
public class ConcurrentRegionOperationsJUnitTest extends DiskRegionTestingBase {

  private int numberOfPutsThreads = 5;

  private int numberOfGetsThreads = 4;

  private int numberOfDestroysThreads = 3;

  private int numberOfClearThreads = 2;

  protected int numberOfForceRollThreads = 3;

  /**
   * ms to run concurrent ops for before signalling them to stop
   */
  protected int TIME_TO_RUN = 1000;

  private boolean exceptionOccurredInPuts = false;

  private boolean exceptionOccurredInGets = false;

  private boolean exceptionOccurredInDestroys = false;

  private boolean exceptionOccurredInClears = false;

  protected boolean exceptionOccurredInForceRolls = false;

  // if this test is to run for a longer time, make this true
  private static final boolean longTest = false;

  protected boolean failure = false;

  private boolean validate;

  protected Region region1;

  private Region region2;

  private final Map<Integer, Lock> map = new ConcurrentHashMap<>();

  private static int counter = 0;

  @Override
  protected final void postSetUp() throws Exception {
    counter++;
    if (longTest) {
      TIME_TO_RUN = 10000;
      numberOfPutsThreads = 5;
      numberOfGetsThreads = 4;
      numberOfDestroysThreads = 3;
      numberOfClearThreads = 2;
      numberOfForceRollThreads = 3;
    }
  }

  @Test
  public void testPersistSyncConcurrency() {
    validate = true;
    DiskRegionProperties p = new DiskRegionProperties();
    p.setRegionName(getName() + counter);
    p.setDiskDirs(dirs);
    p.setCompactionThreshold(99);
    region1 = DiskRegionHelperFactory.getSyncPersistOnlyRegion(cache, p, Scope.LOCAL);
    region2 = concurrencyTest(region1);
    region1 = DiskRegionHelperFactory.getSyncPersistOnlyRegion(cache, p, Scope.LOCAL);
    validate(region1, region2);
  }

  @Test
  public void testPersistAsyncConcurrency() {
    validate = true;
    DiskRegionProperties p = new DiskRegionProperties();
    p.setRegionName(getName() + counter);
    p.setDiskDirs(dirs);
    p.setCompactionThreshold(99);
    p.setBytesThreshold(0);
    p.setTimeInterval(1);
    region1 = DiskRegionHelperFactory.getAsyncPersistOnlyRegion(cache, p);
    region2 = concurrencyTest(region1);
    region1 = DiskRegionHelperFactory.getAsyncPersistOnlyRegion(cache, p);
    validate(region1, region2);
  }

  @Test
  public void testPersistAsyncSmallQueueConcurrency() {
    validate = true;
    DiskRegionProperties p = new DiskRegionProperties();
    p.setRegionName(getName() + counter);
    p.setDiskDirs(dirs);
    p.setCompactionThreshold(99);
    p.setBytesThreshold(1);
    p.setTimeInterval(0);
    region1 = DiskRegionHelperFactory.getAsyncPersistOnlyRegion(cache, p);
    region2 = concurrencyTest(region1);
    region1 = DiskRegionHelperFactory.getAsyncPersistOnlyRegion(cache, p);
    validate(region1, region2);
  }

  @Test
  public void testPersistAndOverflowSyncConcurrency() {
    validate = true;
    DiskRegionProperties p = new DiskRegionProperties();
    p.setRegionName(getName() + counter);
    p.setDiskDirs(dirs);
    p.setCompactionThreshold(99);
    p.setOverFlowCapacity(1);
    region1 = DiskRegionHelperFactory.getSyncOverFlowAndPersistRegion(cache, p);
    region2 = concurrencyTest(region1);
    region1 = DiskRegionHelperFactory.getSyncOverFlowAndPersistRegion(cache, p);
    validate(region1, region2);

  }

  @Test
  public void testPersistAndOverflowAsyncConcurrency() {
    validate = true;
    DiskRegionProperties p = new DiskRegionProperties();
    p.setRegionName(getName() + counter);
    p.setDiskDirs(dirs);
    p.setCompactionThreshold(99);
    p.setBytesThreshold(0);
    p.setTimeInterval(1);
    p.setOverFlowCapacity(1);
    region1 = DiskRegionHelperFactory.getAsyncOverFlowAndPersistRegion(cache, p);
    region2 = concurrencyTest(region1);
    region1 = DiskRegionHelperFactory.getAsyncOverFlowAndPersistRegion(cache, p);
    validate(region1, region2);
  }

  @Test
  public void testPersistAndOverflowAsyncSmallQueueConcurrency() {
    validate = true;
    DiskRegionProperties p = new DiskRegionProperties();
    p.setRegionName(getName() + counter);
    p.setDiskDirs(dirs);
    p.setCompactionThreshold(99);
    p.setBytesThreshold(1);
    p.setTimeInterval(0);
    p.setOverFlowCapacity(1);
    region1 = DiskRegionHelperFactory.getAsyncOverFlowAndPersistRegion(cache, p);
    region2 = concurrencyTest(region1);
    region1 = DiskRegionHelperFactory.getAsyncOverFlowAndPersistRegion(cache, p);
    validate(region1, region2);
  }

  @Test
  public void testNVPersistSyncConcurrency() {
    validate = false;
    DiskRegionProperties p = new DiskRegionProperties();
    p.setRegionName(getName() + counter);
    p.setDiskDirs(dirs);
    p.setCompactionThreshold(99);
    region1 = DiskRegionHelperFactory.getSyncPersistOnlyRegion(cache, p, Scope.LOCAL);
    region2 = concurrencyTest(region1);
    region1 = DiskRegionHelperFactory.getSyncPersistOnlyRegion(cache, p, Scope.LOCAL);
    validate(region1, region2);
  }

  @Test
  public void testNVPersistAsyncConcurrency() {
    validate = false;
    DiskRegionProperties p = new DiskRegionProperties();
    p.setRegionName(getName() + counter);
    p.setDiskDirs(dirs);
    p.setCompactionThreshold(99);
    p.setBytesThreshold(0);
    p.setTimeInterval(1);
    region1 = DiskRegionHelperFactory.getAsyncPersistOnlyRegion(cache, p);
    region2 = concurrencyTest(region1);
    region1 = DiskRegionHelperFactory.getAsyncPersistOnlyRegion(cache, p);
    validate(region1, region2);
  }

  @Test
  public void testNVPersistAsyncSmallQueueConcurrency() {
    validate = false;
    DiskRegionProperties p = new DiskRegionProperties();
    p.setRegionName(getName() + counter);
    p.setDiskDirs(dirs);
    p.setCompactionThreshold(99);
    p.setBytesThreshold(1);
    p.setTimeInterval(0);
    region1 = DiskRegionHelperFactory.getAsyncPersistOnlyRegion(cache, p);
    region2 = concurrencyTest(region1);
    region1 = DiskRegionHelperFactory.getAsyncPersistOnlyRegion(cache, p);
    validate(region1, region2);
  }

  @Test
  public void testNVPersistAndOverflowSyncConcurrency() {
    validate = false;
    DiskRegionProperties p = new DiskRegionProperties();
    p.setRegionName(getName() + counter);
    p.setDiskDirs(dirs);
    p.setCompactionThreshold(100);
    p.setOverFlowCapacity(1);
    region1 = DiskRegionHelperFactory.getSyncOverFlowAndPersistRegion(cache, p);
    region2 = concurrencyTest(region1);
    region1 = DiskRegionHelperFactory.getSyncOverFlowAndPersistRegion(cache, p);
    validate(region1, region2);

  }

  @Test
  public void testNVPersistAndOverflowAsyncConcurrency() {
    validate = false;
    DiskRegionProperties p = new DiskRegionProperties();
    p.setRegionName(getName() + counter);
    p.setDiskDirs(dirs);
    p.setCompactionThreshold(99);
    p.setBytesThreshold(0);
    p.setTimeInterval(1);
    p.setOverFlowCapacity(1);
    region1 = DiskRegionHelperFactory.getAsyncOverFlowAndPersistRegion(cache, p);
    region2 = concurrencyTest(region1);
    region1 = DiskRegionHelperFactory.getAsyncOverFlowAndPersistRegion(cache, p);
    validate(region1, region2);
  }

  @Test
  public void testNVPersistAndOverflowAsyncSmallQueueConcurrency() {
    validate = false;
    DiskRegionProperties p = new DiskRegionProperties();
    p.setRegionName(getName() + counter);
    p.setDiskDirs(dirs);
    p.setCompactionThreshold(99);
    p.setBytesThreshold(1);
    p.setTimeInterval(0);
    p.setOverFlowCapacity(1);
    region1 = DiskRegionHelperFactory.getAsyncOverFlowAndPersistRegion(cache, p);
    region2 = concurrencyTest(region1);
    region1 = DiskRegionHelperFactory.getAsyncOverFlowAndPersistRegion(cache, p);
    validate(region1, region2);
  }

  /**
   * Tests the bug where a get operation on an evicted entry fails to get value as the oplog is
   * deleted by the roller, but the entry was not rolled.
   */
  @Test
  public void testBug35048() {
    DiskRegionProperties p = new DiskRegionProperties();
    p.setMaxOplogSize(1000);
    p.setRegionName(getName() + counter);
    p.setDiskDirs(dirs);
    p.setOverflow(true);
    p.setSynchronous(false);
    p.setOverFlowCapacity(5);
    p.setRolling(true);
    byte[] val = new byte[50];
    region = DiskRegionHelperFactory.getAsyncOverFlowOnlyRegion(cache, p);
    for (int j = 1; j < 6; ++j) {
      region.put("" + j, val);
    }
    // This will overflow the first entry to disk. Its OplogKeyID will be now
    // positive & value in VM is null
    region.put("" + 6, val);
    // Do a get opeartion on entry which has been overflown to disk. The value
    // in VM is now not null, but its Oplog KeyID is positive
    region.get("" + 1);
    // Do a force roll
    region.forceRolling();
    // Do a get on entries starting from 2 to 6 & then 1. This will ensure that
    // the entry 1 gets evicted
    // from memory but is not written to disk as its Oplog KeyId is positive.
    // then do a get operation on entry with key as 1. The vale should be
    // correctly retrieved from
    // Htree implying the roller has rolled the entry correctly
    try {
      region.get("" + 2);
      region.get("" + 3);
      region.get("" + 4);
      region.get("" + 5);
      region.get("" + 6);
      try {
        region.get("" + 1);
      } catch (Exception e) {
        logWriter.severe("Exception occurred  ", e);
        fail(
            "Failed to retrieve value from disk as the Oplog has been rolled but entry still references the Oplog.");
      }

    } catch (Exception e) {
      logWriter.severe("Exception occurred  ", e);
      fail("Test failed because  of unexpected exception");
    }

    // Now force roll the oplog so that the data exists in the Oplog which has
    // been deleted

  }

  @Test
  public void testConcurrentForceRollingAndGetOperation() {
    DiskRegionProperties p = new DiskRegionProperties();
    p.setMaxOplogSize(1000);
    p.setRegionName(getName() + counter);
    p.setDiskDirs(dirs);
    p.setOverflow(true);
    p.setSynchronous(false);
    p.setOverFlowCapacity(5);
    p.setRolling(true);
    byte[] val = new byte[50];
    region = DiskRegionHelperFactory.getAsyncOverFlowOnlyRegion(cache, p);
    for (int j = 1; j < 101; ++j) {
      region.put("" + j, val);
    }
    Thread t1 = new Thread(() -> {
      for (int i = 0; i < 100; ++i) {
        region.forceRolling();

        try {
          Thread.sleep(TIME_TO_RUN / 100);
        } catch (InterruptedException e) {
          fail("interrupted");
        }
      }
    });
    Thread t2 = new Thread(() -> {
      try {
        for (int i = 0; i < 100; ++i) {
          for (int j = 1; j < 101; ++j) {
            region.get("" + j);

          }
        }
      } catch (Exception e) {
        e.printStackTrace();
        failure = true;
      }
    });
    t1.start();
    t2.start();
    ThreadUtils.join(t1, 30 * 1000);
    ThreadUtils.join(t2, 30 * 1000);
    assertTrue(!failure);
  }

  private final AtomicBoolean timeToStop = new AtomicBoolean();

  private boolean isItTimeToStop() {
    return timeToStop.get();
  }

  private CyclicBarrier startLine;

  private void waitForAllStartersToBeReady() {
    try {
      startLine.await();
    } catch (InterruptedException ie) {
      throw new AssertionError("unexpected ", ie);
    } catch (BrokenBarrierException ex) {
      throw new AssertionError("unexpected ", ex);
    }
  }

  private Region concurrencyTest(Region r1) {
    if (validate) {
      for (int i = 0; i < 10; i++) {
        map.put(i, new ReentrantLock());
      }
      region2 =
          cache.createVMRegion("testRegion2", new AttributesFactory().createRegionAttributes());
    }
    startLine = new CyclicBarrier(numberOfPutsThreads + numberOfGetsThreads
        + numberOfDestroysThreads + numberOfClearThreads + numberOfForceRollThreads);
    DoesPuts doesPuts = new DoesPuts();
    DoesGets doesGets = new DoesGets();
    DoesDestroy doesDestroy = new DoesDestroy();
    DoesClear doesClear = new DoesClear();
    DoesForceRoll doesForceRoll = new DoesForceRoll();
    Thread[] putThreads = new Thread[numberOfPutsThreads];
    Thread[] getThreads = new Thread[numberOfGetsThreads];
    Thread[] destroyThreads = new Thread[numberOfDestroysThreads];
    Thread[] clearThreads = new Thread[numberOfClearThreads];
    Thread[] forceRollThreads = new Thread[numberOfForceRollThreads];
    for (int i = 0; i < numberOfPutsThreads; i++) {
      putThreads[i] = new Thread(doesPuts);
      putThreads[i].setName("PutThread" + i);
    }
    for (int i = 0; i < numberOfGetsThreads; i++) {
      getThreads[i] = new Thread(doesGets);
      getThreads[i].setName("GetThread" + i);
    }
    for (int i = 0; i < numberOfDestroysThreads; i++) {
      destroyThreads[i] = new Thread(doesDestroy);
      destroyThreads[i].setName("DelThread" + i);
    }

    for (int i = 0; i < numberOfClearThreads; i++) {
      clearThreads[i] = new Thread(doesClear);
      clearThreads[i].setName("ClearThread" + i);
    }

    for (int i = 0; i < numberOfForceRollThreads; i++) {
      forceRollThreads[i] = new Thread(doesForceRoll);
      forceRollThreads[i].setName("ForceRoll" + i);
    }
    timeToStop.set(false);
    try {
      for (int i = 0; i < numberOfPutsThreads; i++) {
        putThreads[i].start();
      }
      for (int i = 0; i < numberOfGetsThreads; i++) {
        getThreads[i].start();
      }
      for (int i = 0; i < numberOfDestroysThreads; i++) {
        destroyThreads[i].start();
      }
      for (int i = 0; i < numberOfClearThreads; i++) {
        clearThreads[i].start();
      }
      for (int i = 0; i < numberOfForceRollThreads; i++) {
        forceRollThreads[i].start();
      }
      try {
        Thread.sleep(TIME_TO_RUN);
      } catch (InterruptedException e) {
        fail("interrupted");
      }
    } finally {
      timeToStop.set(true);
    }
    for (int i = 0; i < numberOfPutsThreads; i++) {
      ThreadUtils.join(putThreads[i], 60 * 1000);
    }
    for (int i = 0; i < numberOfGetsThreads; i++) {
      ThreadUtils.join(getThreads[i], 60 * 1000);
    }
    for (int i = 0; i < numberOfDestroysThreads; i++) {
      ThreadUtils.join(destroyThreads[i], 60 * 1000);
    }
    for (int i = 0; i < numberOfClearThreads; i++) {
      ThreadUtils.join(clearThreads[i], 60 * 1000);
    }
    for (int i = 0; i < numberOfForceRollThreads; i++) {
      ThreadUtils.join(forceRollThreads[i], 60 * 1000);
    }

    if (validate) {
      Collection entrySet = region2.entrySet();
      Iterator iterator = entrySet.iterator();
      Map.Entry mapEntry = null;
      Object key, value = null;
      ((LocalRegion) r1).getDiskRegion().forceFlush();
      while (iterator.hasNext()) {
        mapEntry = (Map.Entry) iterator.next();
        key = mapEntry.getKey();
        value = mapEntry.getValue();
        if (!(r1.containsKey(key))) {
          fail(" region1 does not contain Key " + key + " but was expected to be there");
        }
        if (!(((LocalRegion) r1).getValueOnDisk(key).equals(value))) {
          fail(" value for key " + key + " is " + ((LocalRegion) r1).getValueOnDisk(key)
              + " which is not consistent, it is supposed to be " + value);
        }
      }
    }
    r1.close();

    if (exceptionOccurredInDestroys) {
      fail("Exception occurred while destroying");
    }
    if (exceptionOccurredInClears) {
      fail("Exception occurred while clearing");
    }

    if (exceptionOccurredInForceRolls) {
      fail("Exception occurred while force Rolling");
    }

    if (exceptionOccurredInGets) {
      fail("Exception occurred while doing gets");
    }

    if (exceptionOccurredInPuts) {
      fail("Exception occurred while doing puts");
    }

    return region2;
  }

  void validate(Region r1, Region r2) {
    if (!validate) {
      return;
    }

    Collection entrySet = r2.entrySet();
    Iterator iterator = entrySet.iterator();
    Map.Entry mapEntry = null;
    Object key, value = null;
    while (iterator.hasNext()) {
      mapEntry = (Map.Entry) iterator.next();
      key = mapEntry.getKey();
      value = mapEntry.getValue();
      if (!(r1.containsKey(key))) {
        fail(" region1 does not contain Key " + key + " but was expected to be there");
      }
      if (!(r1.get(key).equals(value))) {
        fail(" value for key " + key + " is " + r1.get(key)
            + " which is not consistent, it is supposed to be " + value);
      }
    }
    assertEquals(r2.size(), r1.size());

    r1.destroyRegion();
    r2.destroyRegion();
  }

  private final Random random = new Random();

  void put() {
    int randomInt1 = random.nextInt() % 10;
    if (randomInt1 < 0) {
      randomInt1 = randomInt1 * (-1);
    }
    int randomInt2 = random.nextInt() % 100;
    Integer integer1 = randomInt1;
    Integer integer2 = randomInt2;
    Object v = null;
    Object expected = null;
    Lock lock = null;
    if (validate) {
      lock = map.get(integer1);
      lock.lock();
    }
    try {
      try {
        v = region1.put(integer1, integer2);
        if (validate) {
          expected = region2.put(integer1, integer2);
        }
      } catch (Exception e) {
        exceptionOccurredInPuts = true;
        logWriter.severe("Exception occurred in puts ", e);
        fail(" failed during put due to " + e);
      }
    } finally {
      if (lock != null) {
        lock.unlock();
      }
    }
    if (validate) {
      if (v != null) {
        assertEquals(expected, v);
      }
    }
  }

  void get() {
    int randomInt1 = random.nextInt() % 10;
    if (randomInt1 < 0) {
      randomInt1 = randomInt1 * (-1);
    }

    Integer integer1 = randomInt1;
    Object v = null;
    Object expected = null;
    Lock lock = null;
    if (validate) {
      lock = map.get(integer1);
      lock.lock();
    }
    try {
      try {
        v = region1.get(integer1);
        if (validate) {
          expected = region2.get(integer1);
        }
      } catch (Exception e) {
        exceptionOccurredInGets = true;
        logWriter.severe("Exception occurred in get ", e);
        throw new AssertionError(" failed during get due to ", e);
      }
    } finally {
      if (lock != null) {
        lock.unlock();
      }
    }
    if (validate) {
      assertEquals(expected, v);
    }
  }

  void destroy() {
    Exception exceptionOccurred1 = null;
    Exception exceptionOccurred2 = null;
    int randomInt1 = random.nextInt() % 10;
    if (randomInt1 < 0) {
      randomInt1 = randomInt1 * (-1);
    }
    Integer integer1 = randomInt1;
    Object v = null;
    Object expected = null;
    Lock lock = null;
    if (validate) {
      lock = map.get(integer1);
      lock.lock();
    }
    try {
      try {
        v = region1.destroy(integer1);
      } catch (Exception e) {
        exceptionOccurred1 = e;
      }
      if (validate) {
        try {
          expected = region2.destroy(integer1);
        } catch (Exception e) {
          exceptionOccurred2 = e;
        }

        if ((exceptionOccurred1 != null) ^ (exceptionOccurred2 != null)) {
          exceptionOccurredInDestroys = true;
          logWriter.severe("Exception occurred in destroy ex1=" + exceptionOccurred1 + " ex2="
              + exceptionOccurred2);
          fail("Exception occurred in destroy");
        }
      }
    } finally {
      if (lock != null) {
        lock.unlock();
      }
    }
    if (validate) {
      if (v != null) {
        assertEquals(expected, v);
      }
    }
  }

  void clear() {
    if (validate) {
      return; // can't do clear and validate
    }

    try {
      region1.clear();
    } catch (Exception e) {
      exceptionOccurredInClears = true;
      logWriter.severe("Exception occurred in clear=", e);
      throw new AssertionError("Exception occurred in clear", e);
    }
  }

  /**
   * Bug Test for bug # 35139. This bug was occuring because a clear & region destroy operation
   * occurred near concurrently. The region destroy operation notified the roller thread to stop &
   * then it joined with the roller . But by that time clear operation created a new instance of
   * roller thread ( because a clear operation stop/starts the roller) & the destroy operation
   * actually joined with the new thread ( different from the one on which notification was issued
   * to exit).
   */
  @Test
  public void testConcurrentClearAndRegionDestroyBug() {
    DiskRegionProperties p = new DiskRegionProperties();
    p.setMaxOplogSize(10000);
    p.setOverflow(false);
    p.setSynchronous(true);
    p.setOverFlowCapacity(5);
    p.setRolling(true);
    byte[] val = new byte[8000];
    region = DiskRegionHelperFactory.getSyncPersistOnlyRegion(cache, p, Scope.LOCAL);
    region.put("key1", val);
    DiskStoreImpl dimpl = ((LocalRegion) region).getDiskStore();
    LocalRegion.ISSUE_CALLBACKS_TO_CACHE_OBSERVER = true;
    final Thread th = new Thread(() -> region.destroyRegion());

    DiskStoreImpl.DEBUG_DELAY_JOINING_WITH_COMPACTOR = 8000;
    CacheObserver old = CacheObserverHolder.setInstance(new CacheObserverAdapter() {
      boolean skip = false;

      @Override
      public void beforeStoppingCompactor() {
        if (!skip) {
          skip = true;
          th.setPriority(9);
          th.start();
          Thread.yield();
        }
      }

    }

    );

    region.clear();
    ThreadUtils.join(th, 20 * 1000);
    LocalRegion.ISSUE_CALLBACKS_TO_CACHE_OBSERVER = false;
    DiskStoreImpl.DEBUG_DELAY_JOINING_WITH_COMPACTOR = 500;
    CacheObserverHolder.setInstance(old);
  }

  @SuppressWarnings("synthetic-access")
  class DoesPuts implements Runnable {

    @Override
    public void run() {
      waitForAllStartersToBeReady();
      while (!isItTimeToStop()) {
        put();
      }
    }

  }

  @SuppressWarnings("synthetic-access")
  class DoesGets implements Runnable {

    @Override
    public void run() {
      waitForAllStartersToBeReady();
      while (!isItTimeToStop()) {
        get();
      }
    }

  }

  @SuppressWarnings("synthetic-access")
  class DoesDestroy implements Runnable {

    @Override
    public void run() {
      waitForAllStartersToBeReady();
      while (!isItTimeToStop()) {
        destroy();
      }
    }

  }

  @SuppressWarnings("synthetic-access")
  class DoesClear implements Runnable {
    @Override
    public void run() {
      waitForAllStartersToBeReady();
      while (!isItTimeToStop()) {
        try {
          Thread.sleep(TIME_TO_RUN / 100);
        } catch (InterruptedException e) {
          fail("interrupted");
        }
        clear();
      }
    }
  }

  @SuppressWarnings("synthetic-access")
  class DoesForceRoll implements Runnable {

    @Override
    public void run() {
      waitForAllStartersToBeReady();
      while (!isItTimeToStop()) {
        try {
          Thread.sleep(20);
        } catch (InterruptedException e) {
          fail("interrupted");
        }
        forceRoll();
      }
    }

    private void forceRoll() {
      try {
        region1.forceRolling();
      } catch (Exception e) {
        exceptionOccurredInForceRolls = true;
        logWriter.severe("Exception occurred in forceRolling ", e);
        throw new AssertionError(" Exception occurred here", e);
      }
    }

  }

}
