Debugging JGroups large state or farming issues

Issue

• Slow transfer/timing out
• The following messages appear in the logs when state transfer fails

-- MBEANS THAT ARE THE ROOT CAUSE OF THE PROBLEM ---
ObjectName: jboss.cache:service=TomcatClusteringCache
  State: FAILED
  Reason: org.jboss.cache.CacheException: Initial state transfer failed: Channel.getState() returned false
  I Depend On:
    jboss:service=Naming
    jboss:service=TransactionManager
    jboss.aop:service=AspectDeployer
  Depends On Me:
    jboss.web:service=WebServer

2010-07-15 16:00:46,975 WARN  [org.jboss.system.ServiceController] Problem
starting service jboss.cache:service=TomcatClusteringCache
org.jboss.cache.CacheException: Initial state transfer failed: Channel.getState() returned false
        at org.jboss.cache.TreeCache.fetchStateOnStartup(TreeCache.java:3378)
        at org.jboss.cache.TreeCache.startService(TreeCache.java:1556)
        at org.jboss.cache.aop.PojoCache.startService(PojoCache.java:94)
        at org.jboss.system.ServiceMBeanSupport.jbossInternalStart(ServiceMBeanSupport.java:289)
        at org.jboss.system.ServiceMBeanSupport.jbossInternalLifecycle(ServiceMBeanSupport.java:245)
        at sun.reflect.GeneratedMethodAccessor3.invoke(Unknown Source)

• JBoss fails to start with org.jgroups.StateTransferException from JBossCacheManager

ERROR [org.jboss.web.tomcat.service.session.JBossCacheManager.Example] (main) Unable to start manager.
org.jboss.cache.CacheException: java.lang.reflect.InvocationTargetException
    at org.jboss.cache.util.reflect.ReflectionUtil.invokeAccessibly(ReflectionUtil.java:148)
    at org.jboss.cache.factories.ComponentRegistry$PrioritizedMethod.invoke(ComponentRegistry.java:1012)
    at org.jboss.cache.factories.ComponentRegistry.internalStart(ComponentRegistry.java:778)
    at org.jboss.cache.factories.ComponentRegistry.start(ComponentRegistry.java:632)
    at org.jboss.cache.invocation.CacheInvocationDelegate.start(CacheInvocationDelegate.java:345)
    at org.jboss.ha.cachemanager.CacheManagerManagedCache.start(CacheManagerManagedCache.java:104)
    at org.jboss.web.tomcat.service.session.distributedcache.impl.jbc.AbstractJBossCacheService.start(AbstractJBossCacheService.java:203)
    at org.jboss.web.tomcat.service.session.JBossCacheManager.startEmbedded(JBossCacheManager.java:2623)
    at org.jboss.web.tomcat.service.session.JBossCacheManager.start(JBossCacheManager.java:797)
    at org.apache.catalina.core.ContainerBase.setManager(ContainerBase.java:447)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
    at java.lang.reflect.Method.invoke(Method.java:597)
    at org.apache.tomcat.util.modeler.BaseModelMBean.setAttribute(BaseModelMBean.java:414)
    at org.jboss.mx.server.RawDynamicInvoker.setAttribute(RawDynamicInvoker.java:70)
    at org.jboss.mx.server.MBeanServerImpl.setAttribute(MBeanServerImpl.java:617)
    at org.jboss.web.tomcat.service.deployers.TomcatDeployment.performDeployInternal(TomcatDeployment.java:359)
    at org.jboss.web.tomcat.service.deployers.TomcatDeployment.performDeploy(TomcatDeployment.java:145)
    at org.jboss.web.deployers.AbstractWarDeployment.start(AbstractWarDeployment.java:461)
    at org.jboss.web.deployers.WebModule.startModule(WebModule.java:122)
    at org.jboss.web.deployers.WebModule.start(WebModule.java:97)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
    at java.lang.reflect.Method.invoke(Method.java:597)
    at org.jboss.mx.interceptor.ReflectedDispatcher.invoke(ReflectedDispatcher.java:157)
    at org.jboss.mx.server.Invocation.dispatch(Invocation.java:96)
    at org.jboss.mx.server.Invocation.invoke(Invocation.java:88)
    at org.jboss.mx.server.AbstractMBeanInvoker.invoke(AbstractMBeanInvoker.java:264)
    at org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:668)
    at org.jboss.system.microcontainer.ServiceProxy.invoke(ServiceProxy.java:206)
    at $Proxy38.start(Unknown Source)
    at org.jboss.system.microcontainer.StartStopLifecycleAction.installAction(StartStopLifecycleAction.java:42)
    at org.jboss.system.microcontainer.StartStopLifecycleAction.installAction(StartStopLifecycleAction.java:37)
    at org.jboss.dependency.plugins.action.SimpleControllerContextAction.simpleInstallAction(SimpleControllerContextAction.java:62)
    at org.jboss.dependency.plugins.action.AccessControllerContextAction.install(AccessControllerContextAction.java:71)
    at org.jboss.dependency.plugins.AbstractControllerContextActions.install(AbstractControllerContextActions.java:51)
    at org.jboss.dependency.plugins.AbstractControllerContext.install(AbstractControllerContext.java:348)
    at org.jboss.system.microcontainer.ServiceControllerContext.install(ServiceControllerContext.java:297)
    at org.jboss.dependency.plugins.AbstractController.install(AbstractController.java:1652)
    at org.jboss.dependency.plugins.AbstractController.incrementState(AbstractController.java:938)
    at org.jboss.dependency.plugins.AbstractController.resolveContexts(AbstractController.java:1082)
    at org.jboss.dependency.plugins.AbstractController.resolveContexts(AbstractController.java:988)
    at org.jboss.dependency.plugins.AbstractController.change(AbstractController.java:826)
    at org.jboss.dependency.plugins.AbstractController.change(AbstractController.java:556)
    at org.jboss.system.ServiceController.doChange(ServiceController.java:688)
    at org.jboss.system.ServiceController.start(ServiceController.java:460)
    at org.jboss.system.deployers.ServiceDeployer.start(ServiceDeployer.java:163)
    at org.jboss.system.deployers.ServiceDeployer.deploy(ServiceDeployer.java:99)
    at org.jboss.system.deployers.ServiceDeployer.deploy(ServiceDeployer.java:46)
    at org.jboss.deployers.spi.deployer.helpers.AbstractSimpleRealDeployer.internalDeploy(AbstractSimpleRealDeployer.java:62)
    at org.jboss.deployers.spi.deployer.helpers.AbstractRealDeployer.deploy(AbstractRealDeployer.java:55)
    at org.jboss.deployers.plugins.deployers.DeployerWrapper.deploy(DeployerWrapper.java:179)
    at org.jboss.deployers.plugins.deployers.DeployersImpl.doDeploy(DeployersImpl.java:1454)
    at org.jboss.deployers.plugins.deployers.DeployersImpl.doInstallParentFirst(DeployersImpl.java:1172)
    at org.jboss.deployers.plugins.deployers.DeployersImpl.doInstallParentFirst(DeployersImpl.java:1193)
    at org.jboss.deployers.plugins.deployers.DeployersImpl.doInstallParentFirst(DeployersImpl.java:1225)
    at org.jboss.deployers.plugins.deployers.DeployersImpl.install(DeployersImpl.java:1113)
    at org.jboss.dependency.plugins.AbstractControllerContext.install(AbstractControllerContext.java:348)
    at org.jboss.dependency.plugins.AbstractController.install(AbstractController.java:1652)
    at org.jboss.dependency.plugins.AbstractController.incrementState(AbstractController.java:938)
    at org.jboss.dependency.plugins.AbstractController.resolveContexts(AbstractController.java:1082)
    at org.jboss.dependency.plugins.AbstractController.resolveContexts(AbstractController.java:988)
    at org.jboss.dependency.plugins.AbstractController.change(AbstractController.java:826)
    at org.jboss.dependency.plugins.AbstractController.change(AbstractController.java:556)
    at org.jboss.deployers.plugins.deployers.DeployersImpl.process(DeployersImpl.java:789)
    at org.jboss.deployers.plugins.main.MainDeployerImpl.process(MainDeployerImpl.java:699)
    at org.jboss.system.server.profileservice.repository.MainDeployerAdapter.process(MainDeployerAdapter.java:117)
    at org.jboss.system.server.profileservice.repository.ProfileDeployAction.install(ProfileDeployAction.java:70)
    at org.jboss.system.server.profileservice.repository.AbstractProfileAction.install(AbstractProfileAction.java:53)
    at org.jboss.system.server.profileservice.repository.AbstractProfileService.install(AbstractProfileService.java:403)
    at org.jboss.dependency.plugins.AbstractControllerContext.install(AbstractControllerContext.java:348)
    at org.jboss.dependency.plugins.AbstractController.install(AbstractController.java:1652)
    at org.jboss.dependency.plugins.AbstractController.incrementState(AbstractController.java:938)
    at org.jboss.dependency.plugins.AbstractController.resolveContexts(AbstractController.java:1082)
    at org.jboss.dependency.plugins.AbstractController.resolveContexts(AbstractController.java:988)
    at org.jboss.dependency.plugins.AbstractController.install(AbstractController.java:778)
    at org.jboss.dependency.plugins.AbstractController.install(AbstractController.java:543)
    at org.jboss.system.server.profileservice.repository.AbstractProfileService.registerProfile(AbstractProfileService.java:308)
    at org.jboss.system.server.profileservice.ProfileServiceBootstrap.start(ProfileServiceBootstrap.java:256)
    at org.jboss.bootstrap.AbstractServerImpl.start(AbstractServerImpl.java:461)
    at org.jboss.Main.boot(Main.java:221)
    at org.jboss.Main$1.run(Main.java:556)
    at java.lang.Thread.run(Thread.java:662)
Caused by: java.lang.reflect.InvocationTargetException
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
    at java.lang.reflect.Method.invoke(Method.java:597)
    at org.jboss.cache.util.reflect.ReflectionUtil.invokeAccessibly(ReflectionUtil.java:144)
    ... 84 more
Caused by: org.jboss.cache.CacheException: Unable to connect to JGroups channel
    at org.jboss.cache.RPCManagerImpl.start(RPCManagerImpl.java:378)
    ... 89 more
Caused by: org.jgroups.StateTransferException: 192.168.10.10:55200 could not fetch state null from null
    at org.jgroups.JChannel.connect(JChannel.java:464)
    at org.jboss.cache.RPCManagerImpl.start(RPCManagerImpl.java:371)
    ... 89 more
Caused by: org.jgroups.StateTransferException: 192.168.10.10:55200 could not fetch state null from null
    at org.jgroups.JChannel.connect(JChannel.java:459)
    ... 90 more