phoenix-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From 김영우 (YoungWoo Kim) <yw...@apache.org>
Subject Exceptions from queryserver logs
Date Wed, 24 Aug 2016 13:04:42 GMT
Hi,

I've done upgrading Phoenix to 4.8.0-HBase-1.1 our production cluster.
after that, I got the following exceptions from the queryserver logs that
I've never seen before.

Please let me know what should I look into..

Thanks,
Yungwoo


2016-08-23 17:24:15,942 INFO
org.apache.phoenix.iterate.BaseResultIterators: Failed to execute task
during cancel
java.util.concurrent.ExecutionException:
org.apache.phoenix.exception.PhoenixIOException:
org.apache.hadoop.hbase.DoNotRetryIOException:
FDC.BIG_TRACE_SUMMARY,\x0A2016-06-04\x00M10\x00TAP402\x00TMEBE91\x00P9548000B\x00TAP402_CH3_ST2\x00PL_M0_1.5K_NO_BC\x0017\x005\x002016-06-04
11:11:37.4,1467623243434.4e6f6d5bae09ab4f8b62bb251d9c1598.: null
    at org.apache.phoenix.util.ServerUtil.createIOException(ServerUtil.java:87)
    at org.apache.phoenix.util.ServerUtil.throwIOException(ServerUtil.java:53)
    at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$2.nextRaw(BaseScannerRegionObserver.java:405)
    at org.apache.phoenix.iterate.RegionScannerResultIterator.next(RegionScannerResultIterator.java:50)
    at org.apache.phoenix.iterate.OrderedResultIterator.getResultIterator(OrderedResultIterator.java:254)
    at org.apache.phoenix.iterate.OrderedResultIterator.next(OrderedResultIterator.java:198)
    at org.apache.phoenix.coprocessor.ScanRegionObserver.getTopNScanner(ScanRegionObserver.java:324)
    at org.apache.phoenix.coprocessor.ScanRegionObserver.doPostScannerOpen(ScanRegionObserver.java:242)
    at org.apache.phoenix.coprocessor.BaseScannerRegionObserver.postScannerOpen(BaseScannerRegionObserver.java:215)
    at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$52.call(RegionCoprocessorHost.java:1351)
    at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$RegionOperation.call(RegionCoprocessorHost.java:1708)
    at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.execOperation(RegionCoprocessorHost.java:1783)
    at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.execOperationWithResult(RegionCoprocessorHost.java:1747)
    at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.postScannerOpen(RegionCoprocessorHost.java:1346)
    at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:2273)
    at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32385)
    at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2117)
    at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:104)
    at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:133)
    at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:108)
    at java.lang.Thread.run(Thread.java:745)
Caused by: java.nio.BufferUnderflowException
    at java.nio.Buffer.nextGetIndex(Buffer.java:506)
    at java.nio.HeapByteBuffer.getLong(HeapByteBuffer.java:412)
    at org.apache.hadoop.hbase.io.hfile.HFileBlock.toStringHeader(HFileBlock.java:1917)
    at org.apache.hadoop.hbase.io.hfile.ChecksumUtil.validateBlockChecksum(ChecksumUtil.java:157)
    at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.validateBlockChecksum(HFileBlock.java:1746)
    at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1695)
    at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1504)
    at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:437)
    at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:259)
    at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:634)
    at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:614)
    at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:267)
    at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:181)
    at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:387)
    at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:347)
    at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:303)
    at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:268)
    at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:826)
    at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:814)
    at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:648)
    at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:147)
    at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:5517)
    at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:5668)
    at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:5455)
    at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:5441)
    at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$2.nextRaw(BaseScannerRegionObserver.java:375)
    ... 18 more

    at java.util.concurrent.FutureTask.report(FutureTask.java:122)
    at java.util.concurrent.FutureTask.get(FutureTask.java:192)
    at org.apache.phoenix.iterate.BaseResultIterators.close(BaseResultIterators.java:878)
    at org.apache.phoenix.iterate.BaseResultIterators.getIterators(BaseResultIterators.java:815)
    at org.apache.phoenix.iterate.BaseResultIterators.getIterators(BaseResultIterators.java:720)
    at org.apache.phoenix.iterate.MergeSortResultIterator.getMinHeap(MergeSortResultIterator.java:72)
    at org.apache.phoenix.iterate.MergeSortResultIterator.minIterator(MergeSortResultIterator.java:93)
    at org.apache.phoenix.iterate.MergeSortResultIterator.next(MergeSortResultIterator.java:58)
    at org.apache.phoenix.iterate.MergeSortTopNResultIterator.next(MergeSortTopNResultIterator.java:95)
    at org.apache.phoenix.jdbc.PhoenixResultSet.next(PhoenixResultSet.java:778)
    at org.apache.calcite.avatica.jdbc.JdbcResultSet.frame(JdbcResultSet.java:133)
    at org.apache.calcite.avatica.jdbc.JdbcResultSet.create(JdbcResultSet.java:91)
    at org.apache.calcite.avatica.jdbc.JdbcMeta.execute(JdbcMeta.java:867)
    at org.apache.calcite.avatica.remote.LocalService.apply(LocalService.java:268)
    at org.apache.calcite.avatica.remote.Service$ExecuteRequest.accept(Service.java:1024)
    at org.apache.calcite.avatica.remote.Service$ExecuteRequest.accept(Service.java:1000)
    at org.apache.calcite.avatica.remote.AbstractHandler.apply(AbstractHandler.java:95)
    at org.apache.calcite.avatica.remote.ProtobufHandler.apply(ProtobufHandler.java:46)
    at org.apache.calcite.avatica.server.AvaticaProtobufHandler.handle(AvaticaProtobufHandler.java:124)
    at org.apache.phoenix.shaded.org.eclipse.jetty.server.handler.HandlerList.handle(HandlerList.java:52)
    at org.apache.phoenix.shaded.org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97)
    at org.apache.phoenix.shaded.org.eclipse.jetty.server.Server.handle(Server.java:499)
    at org.apache.phoenix.shaded.org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:311)
    at org.apache.phoenix.shaded.org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:257)
    at org.apache.phoenix.shaded.org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:544)
    at org.apache.phoenix.shaded.org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)
    at org.apache.phoenix.shaded.org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555)
    at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.phoenix.exception.PhoenixIOException:
org.apache.hadoop.hbase.DoNotRetryIOException:
FDC.BIG_TRACE_SUMMARY,\x0A2016-06-04\x00M10\x00TAP402\x00TMEBE91\x00P9548000B\x00TAP402_CH3_ST2\x00PL_M0_1.5K_NO_BC\x0017\x005\x002016-06-04
11:11:37.4,1467623243434.4e6f6d5bae09ab4f8b62bb251d9c1598.: null
    at org.apache.phoenix.util.ServerUtil.createIOException(ServerUtil.java:87)
    at org.apache.phoenix.util.ServerUtil.throwIOException(ServerUtil.java:53)
    at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$2.nextRaw(BaseScannerRegionObserver.java:405)
    at org.apache.phoenix.iterate.RegionScannerResultIterator.next(RegionScannerResultIterator.java:50)
    at org.apache.phoenix.iterate.OrderedResultIterator.getResultIterator(OrderedResultIterator.java:254)
    at org.apache.phoenix.iterate.OrderedResultIterator.next(OrderedResultIterator.java:198)
    at org.apache.phoenix.coprocessor.ScanRegionObserver.getTopNScanner(ScanRegionObserver.java:324)
    at org.apache.phoenix.coprocessor.ScanRegionObserver.doPostScannerOpen(ScanRegionObserver.java:242)
    at org.apache.phoenix.coprocessor.BaseScannerRegionObserver.postScannerOpen(BaseScannerRegionObserver.java:215)
    at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$52.call(RegionCoprocessorHost.java:1351)
    at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$RegionOperation.call(RegionCoprocessorHost.java:1708)
    at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.execOperation(RegionCoprocessorHost.java:1783)
    at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.execOperationWithResult(RegionCoprocessorHost.java:1747)
    at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.postScannerOpen(RegionCoprocessorHost.java:1346)
    at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:2273)
    at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32385)
    at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2117)
    at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:104)
    at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:133)
    at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:108)
    at java.lang.Thread.run(Thread.java:745)
Caused by: java.nio.BufferUnderflowException
    at java.nio.Buffer.nextGetIndex(Buffer.java:506)
    at java.nio.HeapByteBuffer.getLong(HeapByteBuffer.java:412)
    at org.apache.hadoop.hbase.io.hfile.HFileBlock.toStringHeader(HFileBlock.java:1917)
    at org.apache.hadoop.hbase.io.hfile.ChecksumUtil.validateBlockChecksum(ChecksumUtil.java:157)
    at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.validateBlockChecksum(HFileBlock.java:1746)
    at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1695)
    at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1504)
    at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:437)
    at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:259)
    at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:634)
    at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:614)
    at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:267)
    at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:181)
    at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:387)
    at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:347)
    at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:303)
    at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:268)
    at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:826)
    at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:814)
    at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:648)
    at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:147)
    at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:5517)
    at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:5668)
    at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:5455)
    at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:5441)
    at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$2.nextRaw(BaseScannerRegionObserver.java:375)
    ... 18 more

    at org.apache.phoenix.util.ServerUtil.parseServerException(ServerUtil.java:111)
    at org.apache.phoenix.iterate.TableResultIterator.initScanner(TableResultIterator.java:174)
    at org.apache.phoenix.iterate.TableResultIterator.next(TableResultIterator.java:124)
    at org.apache.phoenix.iterate.LookAheadResultIterator$1.advance(LookAheadResultIterator.java:47)
    at org.apache.phoenix.iterate.LookAheadResultIterator.init(LookAheadResultIterator.java:59)
    at org.apache.phoenix.iterate.LookAheadResultIterator.peek(LookAheadResultIterator.java:73)
    at org.apache.phoenix.iterate.ParallelIterators$1.call(ParallelIterators.java:121)
    at org.apache.phoenix.iterate.ParallelIterators$1.call(ParallelIterators.java:106)
    at java.util.concurrent.FutureTask.run(FutureTask.java:266)
    at org.apache.phoenix.job.JobManager$InstrumentedJobFutureTask.run(JobManager.java:183)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
    ... 1 more
Caused by: org.apache.hadoop.hbase.DoNotRetryIOException:
org.apache.hadoop.hbase.DoNotRetryIOException:
FDC.BIG_TRACE_SUMMARY,\x0A2016-06-04\x00M10\x00TAP402\x00TMEBE91\x00P9548000B\x00TAP402_CH3_ST2\x00PL_M0_1.5K_NO_BC\x0017\x005\x002016-06-04
11:11:37.4,1467623243434.4e6f6d5bae09ab4f8b62bb251d9c1598.: null
    at org.apache.phoenix.util.ServerUtil.createIOException(ServerUtil.java:87)
    at org.apache.phoenix.util.ServerUtil.throwIOException(ServerUtil.java:53)
    at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$2.nextRaw(BaseScannerRegionObserver.java:405)
    at org.apache.phoenix.iterate.RegionScannerResultIterator.next(RegionScannerResultIterator.java:50)
    at org.apache.phoenix.iterate.OrderedResultIterator.getResultIterator(OrderedResultIterator.java:254)
    at org.apache.phoenix.iterate.OrderedResultIterator.next(OrderedResultIterator.java:198)
    at org.apache.phoenix.coprocessor.ScanRegionObserver.getTopNScanner(ScanRegionObserver.java:324)
    at org.apache.phoenix.coprocessor.ScanRegionObserver.doPostScannerOpen(ScanRegionObserver.java:242)
    at org.apache.phoenix.coprocessor.BaseScannerRegionObserver.postScannerOpen(BaseScannerRegionObserver.java:215)
    at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$52.call(RegionCoprocessorHost.java:1351)
    at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$RegionOperation.call(RegionCoprocessorHost.java:1708)
    at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.execOperation(RegionCoprocessorHost.java:1783)
    at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.execOperationWithResult(RegionCoprocessorHost.java:1747)
    at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.postScannerOpen(RegionCoprocessorHost.java:1346)
    at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:2273)
    at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32385)
    at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2117)
    at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:104)
    at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:133)
    at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:108)
    at java.lang.Thread.run(Thread.java:745)
Caused by: java.nio.BufferUnderflowException
    at java.nio.Buffer.nextGetIndex(Buffer.java:506)
    at java.nio.HeapByteBuffer.getLong(HeapByteBuffer.java:412)
    at org.apache.hadoop.hbase.io.hfile.HFileBlock.toStringHeader(HFileBlock.java:1917)
    at org.apache.hadoop.hbase.io.hfile.ChecksumUtil.validateBlockChecksum(ChecksumUtil.java:157)
    at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.validateBlockChecksum(HFileBlock.java:1746)
    at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1695)
    at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1504)
    at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:437)
    at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:259)
    at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:634)
    at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:614)
    at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:267)
    at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:181)
    at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:387)
    at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:347)
    at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:303)
    at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:268)
    at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:826)
    at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:814)
    at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:648)
    at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:147)
    at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:5517)
    at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:5668)
    at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:5455)
    at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:5441)
    at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$2.nextRaw(BaseScannerRegionObserver.java:375)
    ... 18 more

    at sun.reflect.GeneratedConstructorAccessor52.newInstance(Unknown Source)
    at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
    at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
    at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:106)
    at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:95)
    at org.apache.hadoop.hbase.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:325)
    at org.apache.hadoop.hbase.client.ScannerCallable.openScanner(ScannerCallable.java:381)
    at org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:200)
    at org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:62)
    at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithoutRetries(RpcRetryingCaller.java:200)
    at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:350)
    at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:324)
    at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:126)
    at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:64)
    ... 3 more
Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.DoNotRetryIOException):
org.apache.hadoop.hbase.DoNotRetryIOException:
FDC.BIG_TRACE_SUMMARY,\x0A2016-06-04\x00M10\x00TAP402\x00TMEBE91\x00P9548000B\x00TAP402_CH3_ST2\x00PL_M0_1.5K_NO_BC\x0017\x005\x002016-06-04
11:11:37.4,1467623243434.4e6f6d5bae09ab4f8b62bb251d9c1598.: null
    at org.apache.phoenix.util.ServerUtil.createIOException(ServerUtil.java:87)
    at org.apache.phoenix.util.ServerUtil.throwIOException(ServerUtil.java:53)
    at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$2.nextRaw(BaseScannerRegionObserver.java:405)
    at org.apache.phoenix.iterate.RegionScannerResultIterator.next(RegionScannerResultIterator.java:50)
    at org.apache.phoenix.iterate.OrderedResultIterator.getResultIterator(OrderedResultIterator.java:254)
    at org.apache.phoenix.iterate.OrderedResultIterator.next(OrderedResultIterator.java:198)
    at org.apache.phoenix.coprocessor.ScanRegionObserver.getTopNScanner(ScanRegionObserver.java:324)
    at org.apache.phoenix.coprocessor.ScanRegionObserver.doPostScannerOpen(ScanRegionObserver.java:242)
    at org.apache.phoenix.coprocessor.BaseScannerRegionObserver.postScannerOpen(BaseScannerRegionObserver.java:215)
    at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$52.call(RegionCoprocessorHost.java:1351)
    at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$RegionOperation.call(RegionCoprocessorHost.java:1708)
    at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.execOperation(RegionCoprocessorHost.java:1783)
    at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.execOperationWithResult(RegionCoprocessorHost.java:1747)
    at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.postScannerOpen(RegionCoprocessorHost.java:1346)
    at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:2273)
    at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32385)
    at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2117)
    at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:104)
    at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:133)
    at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:108)
    at java.lang.Thread.run(Thread.java:745)
Caused by: java.nio.BufferUnderflowException
    at java.nio.Buffer.nextGetIndex(Buffer.java:506)
    at java.nio.HeapByteBuffer.getLong(HeapByteBuffer.java:412)
    at org.apache.hadoop.hbase.io.hfile.HFileBlock.toStringHeader(HFileBlock.java:1917)
    at org.apache.hadoop.hbase.io.hfile.ChecksumUtil.validateBlockChecksum(ChecksumUtil.java:157)
    at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.validateBlockChecksum(HFileBlock.java:1746)
    at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1695)
    at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1504)
    at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:437)
    at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:259)
    at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:634)
    at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:614)
    at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:267)
    at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:181)
    at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:387)
    at org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:347)
    at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:303)
    at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:268)
    at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:826)
    at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:814)
    at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:648)
    at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:147)
    at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:5517)
    at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:5668)
    at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:5455)
    at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:5441)
    at org.apache.phoenix.coprocessor.BaseScannerRegionObserver$2.nextRaw(BaseScannerRegionObserver.java:375)
    ... 18 more

    at org.apache.hadoop.hbase.ipc.RpcClientImpl.call(RpcClientImpl.java:1235)
    at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:217)
    at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:318)
    at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$BlockingStub.scan(ClientProtos.java:32831)
    at org.apache.hadoop.hbase.client.ScannerCallable.openScanner(ScannerCallable.java:373)
    ... 10 more

Mime
View raw message