Hello,
Gonna guess that I am somehow at fault here. I am stress testing my app against a cloud event store instance. I sent 500 “create” commands which all successfully complete and the event listeners populate the read model as expected. I have a scheduled task that runs regularly which queries the read model for records meeting a certain criteria and sends another command to the aggregate to perform the next step in the process. The command never makes it to the aggregate and I get this stacktrace. To further complicate things, the scheduled task continues submitting commands in addition to the RetryScheduler I have setup, so it would seem like the system just gets flooded with commands. Please advise.
2024-02-10T23:45:26.886-08:00 WARN 12264 --- [mandProcessor-8] o.a.e.eventstore.AbstractEventStore : Error reading snapshot for aggregate [c7a7fd9d-fd4c-44e7-851b-8c5dae20758b]. Reconstructing from entire event stream.
java.lang.RuntimeException: Was unable to load aggregate due to timeout while waiting for events. Last sequence number received: -1
at io.axoniq.axonserver.connector.event.impl.BufferedAggregateEventStream.hasNext(BufferedAggregateEventStream.java:88) ~[axonserver-connector-java-2023.2.0.jar:2023.2.0]
at io.axoniq.axonserver.connector.event.AggregateEventStream$1.tryAdvance(AggregateEventStream.java:73) ~[axonserver-connector-java-2023.2.0.jar:2023.2.0]
at java.base/java.util.Spliterator.forEachRemaining(Spliterator.java:332) ~[na:na]
at java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:509) ~[na:na]
at java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:499) ~[na:na]
at java.base/java.util.stream.ForEachOps$ForEachOp.evaluateSequential(ForEachOps.java:150) ~[na:na]
at java.base/java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateSequential(ForEachOps.java:173) ~[na:na]
at java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234) ~[na:na]
at java.base/java.util.stream.ReferencePipeline.forEach(ReferencePipeline.java:596) ~[na:na]
at org.axonframework.axonserver.connector.event.axon.AxonServerEventStore$AxonIQEventStorageEngine$1.tryAdvance(AxonServerEventStore.java:690) ~[axon-server-connector-4.9.1.jar:4.9.1]
at java.base/java.util.stream.ReferencePipeline.forEachWithCancel(ReferencePipeline.java:129) ~[na:na]
at java.base/java.util.stream.AbstractPipeline.copyIntoWithCancel(AbstractPipeline.java:527) ~[na:na]
at java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:513) ~[na:na]
at java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:499) ~[na:na]
at java.base/java.util.stream.FindOps$FindOp.evaluateSequential(FindOps.java:150) ~[na:na]
at java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234) ~[na:na]
at java.base/java.util.stream.ReferencePipeline.findFirst(ReferencePipeline.java:647) ~[na:na]
at org.axonframework.eventsourcing.eventstore.AbstractEventStorageEngine.readSnapshot(AbstractEventStorageEngine.java:99) ~[axon-eventsourcing-4.9.1.jar:4.9.1]
at org.axonframework.eventsourcing.eventstore.AbstractEventStore.readEvents(AbstractEventStore.java:80) ~[axon-eventsourcing-4.9.1.jar:4.9.1]
at org.axonframework.axonserver.connector.event.axon.AxonServerEventStore.readEvents(AxonServerEventStore.java:175) ~[axon-server-connector-4.9.1.jar:4.9.1]
at org.axonframework.eventsourcing.EventSourcingRepository.readEvents(EventSourcingRepository.java:171) ~[axon-eventsourcing-4.9.1.jar:4.9.1]
at org.axonframework.eventsourcing.EventSourcingRepository.doLoadWithLock(EventSourcingRepository.java:131) ~[axon-eventsourcing-4.9.1.jar:4.9.1]
at org.axonframework.eventsourcing.EventSourcingRepository.doLoadWithLock(EventSourcingRepository.java:56) ~[axon-eventsourcing-4.9.1.jar:4.9.1]
at org.axonframework.modelling.command.LockingRepository.doLoad(LockingRepository.java:137) ~[axon-modelling-4.9.1.jar:4.9.1]
at org.axonframework.modelling.command.LockingRepository.doLoad(LockingRepository.java:60) ~[axon-modelling-4.9.1.jar:4.9.1]
at org.axonframework.modelling.command.AbstractRepository.lambda$null$5(AbstractRepository.java:151) ~[axon-modelling-4.9.1.jar:4.9.1]
at java.base/java.util.HashMap.computeIfAbsent(HashMap.java:1220) ~[na:na]
at org.axonframework.modelling.command.AbstractRepository.lambda$load$7(AbstractRepository.java:148) ~[axon-modelling-4.9.1.jar:4.9.1]
at org.axonframework.tracing.Span.runSupplier(Span.java:163) ~[axon-messaging-4.9.1.jar:4.9.1]
at org.axonframework.modelling.command.AbstractRepository.load(AbstractRepository.java:145) ~[axon-modelling-4.9.1.jar:4.9.1]
at org.axonframework.modelling.command.AggregateAnnotationCommandHandler$AggregateCommandHandler.handle(AggregateAnnotationCommandHandler.java:568) ~[axon-modelling-4.9.1.jar:4.9.1]
at org.axonframework.modelling.command.AggregateAnnotationCommandHandler$AggregateCommandHandler.handle(AggregateAnnotationCommandHandler.java:557) ~[axon-modelling-4.9.1.jar:4.9.1]
at org.axonframework.messaging.DefaultInterceptorChain.proceed(DefaultInterceptorChain.java:57) ~[axon-messaging-4.9.1.jar:4.9.1]
at org.axonframework.messaging.interceptors.CorrelationDataInterceptor.handle(CorrelationDataInterceptor.java:67) ~[axon-messaging-4.9.1.jar:4.9.1]
at org.axonframework.messaging.DefaultInterceptorChain.proceed(DefaultInterceptorChain.java:55) ~[axon-messaging-4.9.1.jar:4.9.1]
at org.axonframework.messaging.unitofwork.DefaultUnitOfWork.executeWithResult(DefaultUnitOfWork.java:77) ~[axon-messaging-4.9.1.jar:4.9.1]
at org.axonframework.commandhandling.SimpleCommandBus.lambda$handle$2(SimpleCommandBus.java:200) ~[axon-messaging-4.9.1.jar:4.9.1]
at org.axonframework.tracing.Span.runSupplier(Span.java:163) ~[axon-messaging-4.9.1.jar:4.9.1]
at org.axonframework.commandhandling.SimpleCommandBus.handle(SimpleCommandBus.java:191) ~[axon-messaging-4.9.1.jar:4.9.1]
at org.axonframework.commandhandling.SimpleCommandBus.doDispatch(SimpleCommandBus.java:165) ~[axon-messaging-4.9.1.jar:4.9.1]
at org.axonframework.commandhandling.SimpleCommandBus.lambda$dispatch$1(SimpleCommandBus.java:131) ~[axon-messaging-4.9.1.jar:4.9.1]
at org.axonframework.tracing.Span.run(Span.java:101) ~[axon-messaging-4.9.1.jar:4.9.1]
at org.axonframework.commandhandling.SimpleCommandBus.dispatch(SimpleCommandBus.java:125) ~[axon-messaging-4.9.1.jar:4.9.1]
at org.axonframework.axonserver.connector.command.AxonServerCommandBus$CommandProcessingTask.lambda$run$1(AxonServerCommandBus.java:311) ~[axon-server-connector-4.9.1.jar:4.9.1]
at org.axonframework.tracing.Span.run(Span.java:101) ~[axon-messaging-4.9.1.jar:4.9.1]
at org.axonframework.axonserver.connector.command.AxonServerCommandBus$CommandProcessingTask.run(AxonServerCommandBus.java:309) ~[axon-server-connector-4.9.1.jar:4.9.1]
at org.axonframework.axonserver.connector.PriorityRunnable.run(PriorityRunnable.java:58) ~[axon-server-connector-4.9.1.jar:4.9.1]
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[na:na]
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[na:na]
at java.base/java.lang.Thread.run(Thread.java:833) ~[na:na]
Caused by: io.axoniq.axonserver.connector.impl.StreamTimeoutException: Timeout while trying to peek next event from the stream
at io.axoniq.axonserver.connector.impl.FlowControlledBuffer.tryTake(FlowControlledBuffer.java:145) ~[axonserver-connector-java-2023.2.0.jar:2023.2.0]
at io.axoniq.axonserver.connector.event.impl.BufferedAggregateEventStream.hasNext(BufferedAggregateEventStream.java:82) ~[axonserver-connector-java-2023.2.0.jar:2023.2.0]
... 49 common frames omitted