Deadlock issues in client/server mode during massive updates or state-transfer if near caching is used in RHDG 7

Issue

• If content of a cache is massive updated and the connected clients are using near-caching it happen randomly that the clients and servers are not longer responsive
• Client get stuck during rebalancing and servers contain errors within the logfiles
  Most server show messages like this

ERROR [org.infinispan.interceptors.impl.InvocationContextInterceptor] (timeout-thread--p3-t1) ISPN000136: Error executing command PutKeyValueCommand, writing keys [WrappedByteArray{bytes=0x033..., hashCode=-1534131625}]: org.infinispan.util.concurrent.TimeoutException: Timed out waiting for topology 140
    at org.infinispan.interceptors.impl.BaseStateTransferInterceptor$CancellableRetry.run(BaseStateTransferInterceptor.java:309) [infinispan-core-8.5.0.Final-redhat-9.jar:8.5.0.Final-redhat-9]
    at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [rt.jar:1.8.0_181]
    at java.util.concurrent.FutureTask.run(FutureTask.java:266) [rt.jar:1.8.0_181]
    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) [rt.jar:1.8.0_181]
    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) [rt.jar:1.8.0_181]
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [rt.jar:1.8.0_181]
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [rt.jar:1.8.0_181]
    at java.lang.Thread.run(Thread.java:748) [rt.jar:1.8.0_181]

One server shows a different one

ERROR [org.infinispan.interceptors.impl.InvocationContextInterceptor] (HotRod-hotrod-internalServerHandler-6-147) ISPN000136: Error executing command PutKeyValueCommand, writing keys [WrappedByteArray{bytes=0x033..., hashCode=-1534131625}]: org.infinispan.util.concurrent.TimeoutException: ISPN000299: Unable to acquire lock after 10 seconds for key WrappedByteArray{bytes=0x033..., hashCode=-1534131625} and requestor CommandInvocation:rmission-store-14-l0m0k:430907. Lock is held by CommandInvocation:rmission-store-14-l0m0k:430901
    at org.infinispan.util.concurrent.locks.impl.DefaultLockManager$KeyAwareExtendedLockPromise.lock(DefaultLockManager.java:239) [infinispan-core-8.5.0.Final-redhat-9.jar:8.5.0.Final-redhat-9]
    at org.infinispan.interceptors.locking.AbstractLockingInterceptor.lockAndRecord(AbstractLockingInterceptor.java:202) [infinispan-core-8.5.0.Final-redhat-9.jar:8.5.0.Final-redhat-9]
    at org.infinispan.interceptors.locking.AbstractLockingInterceptor.visitNonTxDataWriteCommand(AbstractLockingInterceptor.java:111) [infinispan-core-8.5.0.Final-redhat-9.jar:8.5.0.Final-redhat-9]
    at org.infinispan.interceptors.locking.NonTransactionalLockingInterceptor.visitDataWriteCommand(NonTransactionalLockingInterceptor.java:39) [infinispan-core-8.5.0.Final-redhat-9.jar:8.5.0.Final-redhat-9]
    at org.infinispan.interceptors.locking.AbstractLockingInterceptor.visitPutKeyValueCommand(AbstractLockingInterceptor.java:77) [infinispan-core-8.5.0.Final-redhat-9.jar:8.5.0.Final-redhat-9]
    at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:79) [infinispan-core-8.5.0.Final-redhat-9.jar:8.5.0.Final-redhat-9]
    at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextAndHandle(BaseAsyncInterceptor.java:186) [infinispan-core-8.5.0.Final-redhat-9.jar:8.5.0.Final-redhat-9]
    at org.infinispan.statetransfer.StateTransferInterceptor.handleNonTxWriteCommand(StateTransferInterceptor.java:333) [infinispan-core-8.5.0.Final-redhat-9.jar:8.5.0.Final-redhat-9]
    at org.infinispan.statetransfer.StateTransferInterceptor.handleWriteCommand(StateTransferInterceptor.java:268) [infinispan-core-8.5.0.Final-redhat-9.jar:8.5.0.Final-redhat-9]
    at org.infinispan.statetransfer.StateTransferInterceptor.visitPutKeyValueCommand(StateTransferInterceptor.java:118) [infinispan-core-8.5.0.Final-redhat-9.jar:8.5.0.Final-redhat-9]
    at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:79) [infinispan-core-8.5.0.Final-redhat-9.jar:8.5.0.Final-redhat-9]
    at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextAndFinally(BaseAsyncInterceptor.java:153) [infinispan-core-8.5.0.Final-redhat-9.jar:8.5.0.Final-redhat-9]
    at org.infinispan.interceptors.impl.CacheMgmtInterceptor.updateStoreStatistics(CacheMgmtInterceptor.java:174) [infinispan-core-8.5.0.Final-redhat-9.jar:8.5.0.Final-redhat-9]
    at org.infinispan.interceptors.impl.CacheMgmtInterceptor.visitPutKeyValueCommand(CacheMgmtInterceptor.java:160) [infinispan-core-8.5.0.Final-redhat-9.jar:8.5.0.Final-redhat-9]
    at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:79) [infinispan-core-8.5.0.Final-redhat-9.jar:8.5.0.Final-redhat-9]
    at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNextAndExceptionally(BaseAsyncInterceptor.java:126) [infinispan-core-8.5.0.Final-redhat-9.jar:8.5.0.Final-redhat-9]
    at org.infinispan.interceptors.impl.InvocationContextInterceptor.visitCommand(InvocationContextInterceptor.java:92) [infinispan-core-8.5.0.Final-redhat-9.jar:8.5.0.Final-redhat-9]
    at org.infinispan.interceptors.BaseAsyncInterceptor.invokeNext(BaseAsyncInterceptor.java:59) [infinispan-core-8.5.0.Final-redhat-9.jar:8.5.0.Final-redhat-9]
    at org.infinispan.interceptors.DDAsyncInterceptor.handleDefault(DDAsyncInterceptor.java:53) [infinispan-core-8.5.0.Final-redhat-9.jar:8.5.0.Final-redhat-9]
    at org.infinispan.interceptors.DDAsyncInterceptor.visitPutKeyValueCommand(DDAsyncInterceptor.java:59) [infinispan-core-8.5.0.Final-redhat-9.jar:8.5.0.Final-redhat-9]
    at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:79) [infinispan-core-8.5.0.Final-redhat-9.jar:8.5.0.Final-redhat-9]
    at org.infinispan.interceptors.DDAsyncInterceptor.visitCommand(DDAsyncInterceptor.java:49) [infinispan-core-8.5.0.Final-redhat-9.jar:8.5.0.Final-redhat-9]
    at org.infinispan.interceptors.impl.AsyncInterceptorChainImpl.invoke(AsyncInterceptorChainImpl.java:281) [infinispan-core-8.5.0.Final-redhat-9.jar:8.5.0.Final-redhat-9]
    at org.infinispan.cache.impl.CacheImpl.executeCommandAndCommitIfNeeded(CacheImpl.java:1582) [infinispan-core-8.5.0.Final-redhat-9.jar:8.5.0.Final-redhat-9]
    at org.infinispan.cache.impl.CacheImpl.putInternal(CacheImpl.java:1208) [infinispan-core-8.5.0.Final-redhat-9.jar:8.5.0.Final-redhat-9]
    at org.infinispan.cache.impl.CacheImpl.put(CacheImpl.java:1198) [infinispan-core-8.5.0.Final-redhat-9.jar:8.5.0.Final-redhat-9]
    at org.infinispan.cache.impl.DecoratedCache.put(DecoratedCache.java:549) [infinispan-core-8.5.0.Final-redhat-9.jar:8.5.0.Final-redhat-9]
    at org.infinispan.cache.impl.AbstractDelegatingAdvancedCache.put(AbstractDelegatingAdvancedCache.java:305) [infinispan-core-8.5.0.Final-redhat-9.jar:8.5.0.Final-redhat-9]
    at org.infinispan.cache.impl.EncoderCache.put(EncoderCache.java:451) [infinispan-core-8.5.0.Final-redhat-9.jar:8.5.0.Final-redhat-9]
    at org.infinispan.cache.impl.AbstractDelegatingAdvancedCache.put(AbstractDelegatingAdvancedCache.java:305) [infinispan-core-8.5.0.Final-redhat-9.jar:8.5.0.Final-redhat-9]
    at org.infinispan.server.hotrod.CacheDecodeContext.put(CacheDecodeContext.java:424) [infinispan-server-hotrod-8.5.0.Final-redhat-9.jar:8.5.0.Final-redhat-9]
    at org.infinispan.server.hotrod.ContextHandler.realRead(ContextHandler.java:66) [infinispan-server-hotrod-8.5.0.Final-redhat-9.jar:8.5.0.Final-redhat-9]
    at org.infinispan.server.hotrod.ContextHandler.lambda$channelRead0$0(ContextHandler.java:53) [infinispan-server-hotrod-8.5.0.Final-redhat-9.jar:8.5.0.Final-redhat-9]
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [rt.jar:1.8.0_181]
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [rt.jar:1.8.0_181]
    at io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:144) [netty-all-4.1.8.Final-redhat-2.jar:4.1.8.Final-redhat-2]
    at java.lang.Thread.run(Thread.java:748) [rt.jar:1.8.0_181]