org.apache.hadoop.hbase.ipc.Call

https://www.cnblogs.com/parent-absent-son/p/9932260.html


org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=36, exceptions:

org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=36, exceptions:
Thu Mar 12 00:15:49 CST 2020, null, java.net.SocketTimeoutException: callTimeout=60000, callDuration=60304: row '11' on table 'dw_app' at region=dw_app,11,1537153390542.d1fe5dbbc180e3b62ed9634ce9b6fc02., hostname=xxxxxxxxxxxxxx.xxxx.com.cn,16020,1571111359242, seqNum=30134436

    at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:276)
    at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:207)
    at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:60)
    at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithoutRetries(RpcRetryingCaller.java:200)
    at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:320)
    at org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:403)
    at org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:364)
    at org.apache.hadoop.hbase.mapreduce.TableRecordReaderImpl.nextKeyValue(TableRecordReaderImpl.java:254)
    at org.apache.hadoop.hbase.mapreduce.TableRecordReader.nextKeyValue(TableRecordReader.java:147)
    at org.apache.hadoop.hbase.mapreduce.TableInputFormatBase$1.nextKeyValue(TableInputFormatBase.java:216)
    at org.apache.spark.rdd.NewHadoopRDD$$anon$1.hasNext(NewHadoopRDD.scala:179)
    at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:39)
    at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327)
    at org.apache.spark.shuffle.sort.BypassMergeSortShuffleWriter.write(BypassMergeSortShuffleWriter.java:126)
    at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:73)
    at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:41)
    at org.apache.spark.scheduler.Task.run(Task.scala:89)
    at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:247)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
    at java.lang.Thread.run(Thread.java:748)
Caused by: java.net.SocketTimeoutException: callTimeout=60000, callDuration=60304: row '11' on table 'dw_app' at region=dw_app,11,1537153390542.d1fe5dbbc180e3b62ed9634ce9b6fc02., hostname=xxxxxxxxxxxxxx.xxxx.com.cn,16020,1571111359242, seqNum=30134436
    at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:159)
    at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:65)
    ... 3 more
Caused by: java.io.IOException: Call to xxxxxxxxxxxxxx.xxxx.com.cn/xx.xx.xx.xx:16020 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call id=5, waitTime=60001, operationTimeout=60000 expired.
    at org.apache.hadoop.hbase.ipc.AbstractRpcClient.wrapException(AbstractRpcClient.java:291)
    at org.apache.hadoop.hbase.ipc.RpcClientImpl.call(RpcClientImpl.java:1272)
    at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:226)
    at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:331)
    at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$BlockingStub.scan(ClientProtos.java:34094)
    at org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:219)
    at org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:64)
    at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithoutRetries(RpcRetryingCaller.java:200)
    at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:360)
    at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:334)
    at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:126)
    ... 4 more
Caused by: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call id=5, waitTime=60001, operationTimeout=60000 expired.
    at org.apache.hadoop.hbase.ipc.Call.checkAndSetTimeout(Call.java:73)
    at org.apache.hadoop.hbase.ipc.RpcClientImpl.call(RpcClientImpl.java:1246)
    ... 13 more

   conf.setLong("hbase.rpc.timeout", 600000)
    conf.setLong("hbase.client.operation.timeout", 600000)
    conf.setLong("hbase.client.scanner.timeout.period", 600000)
    conf.setLong("hbase.regionserver.lease.period", 600000)
    // ReusablePool，RoundRobinPool，ThreadLocalPool
    conf.set("hbase.client.ipc.pool.type","ThreadLocalPool")
    conf.setInt("hbase.client.ipc.pool.size",10)