MQ broker crash

Hi,

We are using ActiveMQ 5.5.1 and recently the broker crashed. It failed to
restart until we cleared all the data files.

Any help to understand the rootcause or workaround to handle this issue ?

Below are the exceptions noticed in activemq.log,

2013-12-21 20:33:16,207 | ERROR | KahaDB failed to store to Journal |
org.apache.activemq.store.kahadb.MessageDatabase |
ConcurrentQueueStoreAndDispatch
java.io.EOFException: Chunk stream does not exist at page: 9449
at
org.apache.kahadb.page.Transaction$2.readPage(Transaction.java:454)
at org.apache.kahadb.page.Transaction$2.<init>(Transaction.java:431)
at
org.apache.kahadb.page.Transaction.openInputStream(Transaction.java:428)
at org.apache.kahadb.page.Transaction.load(Transaction.java:404)
at org.apache.kahadb.page.Transaction.load(Transaction.java:361)
at org.apache.kahadb.index.BTreeIndex.loadNode(BTreeIndex.java:262)
at org.apache.kahadb.index.BTreeNode.getChild(BTreeNode.java:225)
at org.apache.kahadb.index.BTreeNode.getLeafNode(BTreeNode.java:672)
at org.apache.kahadb.index.BTreeNode.put(BTreeNode.java:368)
at org.apache.kahadb.index.BTreeIndex.put(BTreeIndex.java:189)
at
org.apache.activemq.store.kahadb.MessageDatabase.upadateIndex(MessageDatabase.java:993)
at
org.apache.activemq.store.kahadb.MessageDatabase$14.execute(MessageDatabase.java:872)
at org.apache.kahadb.page.Transaction.execute(Transaction.java:728)
at
org.apache.activemq.store.kahadb.MessageDatabase.process(MessageDatabase.java:870)
at
org.apache.activemq.store.kahadb.MessageDatabase$13.visit(MessageDatabase.java:828)
at
org.apache.activemq.store.kahadb.data.KahaAddMessageCommand.visit(KahaAddMessageCommand.java:241)
at
org.apache.activemq.store.kahadb.MessageDatabase.process(MessageDatabase.java:825)
at
org.apache.activemq.store.kahadb.MessageDatabase.store(MessageDatabase.java:751)
at
org.apache.activemq.store.kahadb.KahaDBStore$KahaDBMessageStore.addMessage(KahaDBStore.java:366)
at
org.apache.activemq.store.kahadb.KahaDBStore$StoreQueueTask.run(KahaDBStore.java:1123)
at
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:619)

2013-12-21 20:33:15,897 | WARN | Async error occurred:
java.lang.ClassCastException: java.lang.Long cannot be cast to
java.lang.String | org.apache.activemq.broker.TransportConnection.Service |
ActiveMQ Transport: tcp:///10.128.24.198:5
4154
java.lang.ClassCastException: java.lang.Long cannot be cast to
java.lang.String
at
org.apache.kahadb.util.StringMarshaller.writePayload(StringMarshaller.java:28)
at
org.apache.kahadb.index.BTreeNode$Marshaller.writePayload(BTreeNode.java:163)
at
org.apache.kahadb.index.BTreeNode$Marshaller.writePayload(BTreeNode.java:147)
at org.apache.kahadb.page.Transaction.store(Transaction.java:249)
at org.apache.kahadb.index.BTreeIndex.storeNode(BTreeIndex.java:289)
at org.apache.kahadb.index.BTreeNode.remove(BTreeNode.java:316)
at org.apache.kahadb.index.BTreeNode.remove(BTreeNode.java:287)
at org.apache.kahadb.index.BTreeNode.remove(BTreeNode.java:287)
at org.apache.kahadb.index.BTreeIndex.remove(BTreeIndex.java:194)
at
org.apache.activemq.store.kahadb.MessageDatabase.updateIndex(MessageDatabase.java:1029)
at
org.apache.activemq.store.kahadb.MessageDatabase$15.execute(MessageDatabase.java:890)
at org.apache.kahadb.page.Transaction.execute(Transaction.java:728)
at
org.apache.activemq.store.kahadb.MessageDatabase.process(MessageDatabase.java:888)
at
org.apache.activemq.store.kahadb.MessageDatabase$13.visit(MessageDatabase.java:833)
at
org.apache.activemq.store.kahadb.data.KahaRemoveMessageCommand.visit(KahaRemoveMessageCommand.java:220)
at
org.apache.activemq.store.kahadb.MessageDatabase.process(MessageDatabase.java:825)
at
org.apache.activemq.store.kahadb.MessageDatabase.store(MessageDatabase.java:751)
at
org.apache.activemq.store.kahadb.KahaDBStore$KahaDBMessageStore.removeMessage(KahaDBStore.java:378)
at
org.apache.activemq.store.kahadb.KahaDBStore$KahaDBMessageStore.removeAsyncMessage(KahaDBStore.java:350)
at
org.apache.activemq.store.kahadb.KahaDBTransactionStore.removeAsyncMessage(KahaDBTransactionStore.java:457)
at
org.apache.activemq.store.kahadb.KahaDBTransactionStore$1.removeAsyncMessage(KahaDBTransactionStore.java:171)
at
org.apache.activemq.broker.region.Queue.acknowledge(Queue.java:800)
at
org.apache.activemq.broker.region.Queue.removeMessage(Queue.java:1530)
at
org.apache.activemq.broker.region.QueueSubscription.acknowledge(QueueSubscription.java:59)
at
org.apache.activemq.broker.region.PrefetchSubscription.acknowledge(PrefetchSubscription.java:253)
at
org.apache.activemq.broker.region.AbstractRegion.acknowledge(AbstractRegion.java:383)
at
org.apache.activemq.broker.region.RegionBroker.acknowledge(RegionBroker.java:546)
at
org.apache.activemq.broker.BrokerFilter.acknowledge(BrokerFilter.java:78)
at
org.apache.activemq.broker.BrokerFilter.acknowledge(BrokerFilter.java:78)
at
org.apache.activemq.broker.TransactionBroker.acknowledge(TransactionBroker.java:197)
at
org.apache.activemq.broker.MutableBrokerFilter.acknowledge(MutableBrokerFilter.java:87)
at
org.apache.activemq.broker.TransportConnection.processMessageAck(TransportConnection.java:468)
at org.apache.activemq.command.MessageAck.visit(MessageAck.java:214)
at
org.apache.activemq.broker.TransportConnection.service(TransportConnection.java:310)
at
org.apache.activemq.broker.TransportConnection$1.onCommand(TransportConnection.java:184)
at
org.apache.activemq.transport.TransportFilter.onCommand(TransportFilter.java:69)
at
org.apache.activemq.transport.WireFormatNegotiator.onCommand(WireFormatNegotiator.java:113)
at
org.apache.activemq.transport.InactivityMonitor.onCommand(InactivityMonitor.java:227)
at
org.apache.activemq.transport.TransportSupport.doConsume(TransportSupport.java:83)
at
org.apache.activemq.transport.tcp.TcpTransport.doRun(TcpTransport.java:222)
at
org.apache.activemq.transport.tcp.TcpTransport.run(TcpTransport.java:204)
at java.lang.Thread.run(Thread.java:619)