ConcurrentModificationException when use async

Dear all, Our team is facing a very strange issue like this: We defined html dialog with 3 p:remotecommands with async flag is set to true. And occationally we got the ConcurrentModificationException when access to that page. Till now, we didn't find any effective way to reproduce it. It happens on Ivy Engine version 7.1

May you have a look and support us if you find out something from it. Thank you so much.

Here is the log. The function loadOpenAmountChart is one of 3 listeners of the remoteCommands

ERROR [runtimelog.luz.luz_finance.html_dialog] [https-jsse-nio-8443-exec-10] [request=/ivy/faces/instances/luz/luz_finance$2/1685452B4979A3AB/ch.klara.luz.fin.partner.PartnerOverview/PartnerOverview.xhtml, session=41 (diem.nguyen@axonactive.com), task=973532, application=1, requestId=581, executionContext=SYSTEM, pmv=luz$luz_finance$2, hd=ch.klara.luz.fin.partner.PartnerOverview] Error during rendering of html dialog 'ch.klara.luz.fin.partner.PartnerOverview' org.apache.myfaces.view.facelets.el.ContextAwareELException: javax.el.ELException: java.util.ConcurrentModificationException at org.apache.myfaces.view.facelets.el.ContextAwareTagMethodExpression.invoke(ContextAwareTagMethodExpression.java:108) at javax.faces.event.MethodExpressionActionListener.processAction(MethodExpressionActionListener.java:81) at javax.faces.event.ActionEvent.processListener(ActionEvent.java:48) at javax.faces.component.UIComponentBase.broadcast(UIComponentBase.java:429) at javax.faces.component.UICommand.broadcast(UICommand.java:103) at javax.faces.component.UIViewRoot._broadcastAll(UIViewRoot.java:1174) at javax.faces.component.UIViewRoot.broadcastEvents(UIViewRoot.java:365) at javax.faces.component.UIViewRoot._process(UIViewRoot.java:1660) at javax.faces.component.UIViewRoot.processApplication(UIViewRoot.java:864) at org.apache.myfaces.lifecycle.InvokeApplicationExecutor.execute(InvokeApplicationExecutor.java:42) at org.apache.myfaces.lifecycle.LifecycleImpl.executePhase(LifecycleImpl.java:196) at org.apache.myfaces.lifecycle.LifecycleImpl.execute(LifecycleImpl.java:143) at javax.faces.webapp.FacesServlet.service(FacesServlet.java:198) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:231) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) at org.apache.catalina.filters.HttpHeaderSecurityFilter.doFilter(HttpHeaderSecurityFilter.java:124) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) at io.prismic.servlet.PrismicFilter.doFilter(PrismicFilter.java:43) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) at ch.ivyteam.ivy.dialog.execution.jsf.controller.rest.IvyRestServiceFilter.doFilter(IvyRestServiceFilter.java:63) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) at ch.ivyteam.ivy.webserver.restricted.jsf.IvyFacesFilter.chainDialogRequest(IvyFacesFilter.java:100) at ch.ivyteam.ivy.webserver.restricted.jsf.IvyFacesFilter.filter(IvyFacesFilter.java:68) at ch.ivyteam.ivy.webserver.restricted.jsf.IvyFacesFilter.doFilter(IvyFacesFilter.java:54) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) at org.apache.catalina.filters.SetCharacterEncodingFilter.doFilter(SetCharacterEncodingFilter.java:108) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) at ch.ivyteam.ivy.webserver.internal.exception.IvyExceptionFilter.doFilter(IvyExceptionFilter.java:49) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) at ch.ivyteam.ivy.webserver.internal.IvyExecuteAsSystemFilter$1.call(IvyExecuteAsSystemFilter.java:45) at ch.ivyteam.ivy.webserver.internal.IvyExecuteAsSystemFilter$1.call(IvyExecuteAsSystemFilter.java:1) at ch.ivyteam.util.callable.AbstractExecutionContext.callInContext(AbstractExecutionContext.java:16) at ch.ivyteam.ivy.security.internal.SecurityManager.executeAsSystem(SecurityManager.java:1128) at ch.ivyteam.ivy.webserver.internal.IvyExecuteAsSystemFilter.doFilter(IvyExecuteAsSystemFilter.java:39) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) at ch.ivyteam.ivy.webserver.internal.startup.IvyEngineUnavailableFilter.doFilter(IvyEngineUnavailableFilter.java:46) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) at ch.ivyteam.ivy.webserver.internal.filter.IvyCurrentHttpRequestFilter.doFilter(IvyCurrentHttpRequestFilter.java:39) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:89) at com.google.inject.servlet.ManagedFilterPipeline.dispatch(ManagedFilterPipeline.java:120) at com.google.inject.servlet.GuiceFilter.doFilter(GuiceFilter.java:135) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:198) at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:96) at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:496) at com.axonivy.auth.valve.AuthenticationValve.invoke(AuthenticationValve.java:35) at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:140) at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:81) at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:87) at ch.ivyteam.ivy.webserver.internal.PerformanceLogValve.invoke(PerformanceLogValve.java:55) at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:342) at org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:803) at org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:66) at org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:790) at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1459) at org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61) at java.lang.Thread.run(Thread.java:745) Caused by: javax.el.ELException: java.util.ConcurrentModificationException at org.apache.el.parser.AstValue.invoke(AstValue.java:260) at org.apache.el.MethodExpressionImpl.invoke(MethodExpressionImpl.java:267) at ch.ivyteam.ivy.dialog.execution.jsf.controller.el.MethodExpressionWraper.invoke(MethodExpressionWraper.java:29) at ch.ivyteam.ivy.dialog.execution.jsf.controller.el.CompositeComponentAwareMethodExpression.invoke(CompositeComponentAwareMethodExpression.java:55) at org.apache.myfaces.view.facelets.el.ContextAwareTagMethodExpression.invoke(ContextAwareTagMethodExpression.java:96) ... 70 more Caused by: java.util.ConcurrentModificationException at java.util.LinkedHashMap$LinkedHashIterator.nextNode(LinkedHashMap.java:719) at java.util.LinkedHashMap$LinkedKeyIterator.next(LinkedHashMap.java:742) at java.util.AbstractCollection.finishToArray(AbstractCollection.java:232) at java.util.AbstractCollection.toArray(AbstractCollection.java:143) at java.util.ArrayList.<init>(ArrayList.java:177) at ch.ivyteam.ivy.dialog.execution.DialogRuntimeLogicHelper.getMethodSignatures(DialogRuntimeLogicHelper.java:151) at ch.ivyteam.ivy.dialog.execution.DialogRuntimeLogicHelper.findMethodSignature(DialogRuntimeLogicHelper.java:97) at ch.ivyteam.ivy.dialog.execution.DialogRuntime.findMethodSignature(DialogRuntime.java:299) at ch.ivyteam.ivy.dialog.execution.jsf.controller.IvyLogicBeanCreator$LogicBeanInvocationHandler.invoke(IvyLogicBeanCreator.java:47) at com.sun.proxy.$Proxy178.loadOpenAmountChart(Unknown Source) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.el.parser.AstValue.invoke(AstValue.java:247) ... 74 more 02:42:14.618 ERROR [org.primefaces.application.exceptionhandler.PrimeExceptionHandler] [https-jsse-nio-8443-exec-10] [request=/ivy/faces/instances/luz/luz_finance$2/1685452B4979A3AB/ch.klara.luz.fin.partner.PartnerOverview/PartnerOverview.xhtml, session=41 (diem.nguyen@axonactive.com), task=973532, application=1, requestId=581, executionContext=SYSTEM, pmv=luz$luz_finance$2, hd=ch.klara.luz.fin.partner.PartnerOverview] null java.util.ConcurrentModificationException at java.util.LinkedHashMap$LinkedHashIterator.nextNode(LinkedHashMap.java:719) at java.util.LinkedHashMap$LinkedKeyIterator.next(LinkedHashMap.java:742) at java.util.AbstractCollection.finishToArray(AbstractCollection.java:232) at java.util.AbstractCollection.toArray(AbstractCollection.java:143) at java.util.ArrayList.<init>(ArrayList.java:177) at ch.ivyteam.ivy.dialog.execution.DialogRuntimeLogicHelper.getMethodSignatures(DialogRuntimeLogicHelper.java:151) at ch.ivyteam.ivy.dialog.execution.DialogRuntimeLogicHelper.findMethodSignature(DialogRuntimeLogicHelper.java:97) at ch.ivyteam.ivy.dialog.execution.DialogRuntime.findMethodSignature(DialogRuntime.java:299) at ch.ivyteam.ivy.dialog.execution.jsf.controller.IvyLogicBeanCreator$LogicBeanInvocationHandler.invoke(IvyLogicBeanCreator.java:47) at com.sun.proxy.$Proxy178.loadOpenAmountChart(Unknown Source) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.el.parser.AstValue.invoke(AstValue.java:247) at org.apache.el.MethodExpressionImpl.invoke(MethodExpressionImpl.java:267) at ch.ivyteam.ivy.dialog.execution.jsf.controller.el.MethodExpressionWraper.invoke(MethodExpressionWraper.java:29) at ch.ivyteam.ivy.dialog.execution.jsf.controller.el.CompositeComponentAwareMethodExpression.invoke(CompositeComponentAwareMethodExpression.java:55) at org.apache.myfaces.view.facelets.el.ContextAwareTagMethodExpression.invoke(ContextAwareTagMethodExpression.java:96) at javax.faces.event.MethodExpressionActionListener.processAction(MethodExpressionActionListener.java:81) at javax.faces.event.ActionEvent.processListener(ActionEvent.java:48) at javax.faces.component.UIComponentBase.broadcast(UIComponentBase.java:429) at javax.faces.component.UICommand.broadcast(UICommand.java:103) at javax.faces.component.UIViewRoot._broadcastAll(UIViewRoot.java:1174) at javax.faces.component.UIViewRoot.broadcastEvents(UIViewRoot.java:365) at javax.faces.component.UIViewRoot._process(UIViewRoot.java:1660) at javax.faces.component.UIViewRoot.processApplication(UIViewRoot.java:864) at org.apache.myfaces.lifecycle.InvokeApplicationExecutor.execute(InvokeApplicationExecutor.java:42) at org.apache.myfaces.lifecycle.LifecycleImpl.executePhase(LifecycleImpl.java:196) at org.apache.myfaces.lifecycle.LifecycleImpl.execute(LifecycleImpl.java:143) at javax.faces.webapp.FacesServlet.service(FacesServlet.java:198) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:231) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) at org.apache.catalina.filters.HttpHeaderSecurityFilter.doFilter(HttpHeaderSecurityFilter.java:124) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) at io.prismic.servlet.PrismicFilter.doFilter(PrismicFilter.java:43) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) at ch.ivyteam.ivy.dialog.execution.jsf.controller.rest.IvyRestServiceFilter.doFilter(IvyRestServiceFilter.java:63) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) at ch.ivyteam.ivy.webserver.restricted.jsf.IvyFacesFilter.chainDialogRequest(IvyFacesFilter.java:100) at ch.ivyteam.ivy.webserver.restricted.jsf.IvyFacesFilter.filter(IvyFacesFilter.java:68) at ch.ivyteam.ivy.webserver.restricted.jsf.IvyFacesFilter.doFilter(IvyFacesFilter.java:54) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) at org.apache.catalina.filters.SetCharacterEncodingFilter.doFilter(SetCharacterEncodingFilter.java:108) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) at ch.ivyteam.ivy.webserver.internal.exception.IvyExceptionFilter.doFilter(IvyExceptionFilter.java:49) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) at ch.ivyteam.ivy.webserver.internal.IvyExecuteAsSystemFilter$1.call(IvyExecuteAsSystemFilter.java:45) at ch.ivyteam.ivy.webserver.internal.IvyExecuteAsSystemFilter$1.call(IvyExecuteAsSystemFilter.java:1) at ch.ivyteam.util.callable.AbstractExecutionContext.callInContext(AbstractExecutionContext.java:16) at ch.ivyteam.ivy.security.internal.SecurityManager.executeAsSystem(SecurityManager.java:1128) at ch.ivyteam.ivy.webserver.internal.IvyExecuteAsSystemFilter.doFilter(IvyExecuteAsSystemFilter.java:39) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) at ch.ivyteam.ivy.webserver.internal.startup.IvyEngineUnavailableFilter.doFilter(IvyEngineUnavailableFilter.java:46) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) at ch.ivyteam.ivy.webserver.internal.filter.IvyCurrentHttpRequestFilter.doFilter(IvyCurrentHttpRequestFilter.java:39) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:89) at com.google.inject.servlet.ManagedFilterPipeline.dispatch(ManagedFilterPipeline.java:120) at com.google.inject.servlet.GuiceFilter.doFilter(GuiceFilter.java:135) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:198) at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:96) at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:496) at com.axonivy.auth.valve.AuthenticationValve.invoke(AuthenticationValve.java:35) at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:140) at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:81) at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:87) at ch.ivyteam.ivy.webserver.internal.PerformanceLogValve.invoke(PerformanceLogValve.java:55) at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:342) at org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:803) at org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:66) at org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:790) at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1459) at org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61) at java.lang.Thread.run(Thread.java:745)