flink run --detached --target yarn-per-job --classpath file:///home/sysop/compute-impl-1.0-jar-with-dependencies-70254400500797440.jar -Dparallelism.default=1 -Djobmanager.memory.process.size=1600m -Dtaskmanager.memory.process.size=1728m -Dyarn.application.type=isop-j-s -Dyarn.application.name=compute-70274432677056512 /home/sysop/model-compute-1.0.0-jar-with-dependencies.jar --topic hello-world --conf eyJzdXBlclBhcmFtcyI6W10sImlucHV0UGFyYW1zIjoiW3tcInNvdXJjZVwiOiBcIjFcIiwgXCJkZXNjcmliZVwiOiBcIuaPj+i/sFwiLCBcInJlcXVpcmVkXCI6IFwiMVwiLCBcInBhcmFtX25hbWVcIjogXCJwYXJhbTFcIiwgXCJwYXJhbV90eXBlXCI6IFwiU3RyaW5nXCIsIFwic291cmNlX3ZhbHVlXCI6IFwic2RzLnNkYTA5OFwifV0iLCJvdXRQYXJhbXMiOiJ7XCJyZXN1bHRfYmluZFwiOiB7XCJhbGVydF9uYW1lXCI6XCLov57nu60z5aSpXCIsXCJkZWZlY3RfZ3JhZGVcIjpbe1wicGFyYW1fbmFtZVwiOlwicXVleGlhbmRlbmdqaVwiLFwiZGVzY3JpYmVcIjpcIuiuvue8uumZt+etiee6p1wiLFwicGFyYW1fdHlwZVwiOlwiU3RyaW5nXCJ9XSxcImVxdWlwX25hbWVfZW5cIjpbe1wicGFyYW1fbmFtZVwiOlwic2hlYmVpXCIsXCJkZXNjcmliZVwiOlwi6K6+5aSH6Iux5paH5ZCNXCIsXCJwYXJhbV90eXBlXCI6XCJTdHJpbmdcIn1dLFwic3RhdGlvbl9pZFwiOlt7XCJwYXJhbV9uYW1lXCI6XCJiaWFuZGlhbnpoYW5cIixcImRlc2NyaWJlXCI6XCLlj5jnlLXnq5noi7HmloflkI1cIixcInBhcmFtX3R5cGVcIjpcIlN0cmluZ1wifV0sXCJ0XCI6W3tcInBhcmFtX25hbWVcIjpcInNoaWphblwiLFwiZGVzY3JpYmVcIjpcIuaXtumXtFwiLFwicGFyYW1fdHlwZVwiOlwiRG91YmxlXCJ9XSxcInZcIjpbe1wicGFyYW1fbmFtZVwiOlwiemhpXCIsXCJkZXNjcmliZVwiOlwi5YC8XCIsXCJwYXJhbV90eXBlXCI6XCJTdHJpbmdcIn1dfSwgXCJyZXN1bHRfc2lua1wiOiBcIjNcIn0iLCJkYXRhc291cmNlQ29uZmlnIjpbeyJuYW1lIjoicHVsc2FyIiwidmFsdWUiOnsiYWRtaW5VcmwiOiJodHRwOi8vMTAuMi4yLjEzOTo4MDgwIiwiZ3JvdXBOYW1lIjoiYWxnb3JpdGhtIiwic2VydmljZVVybCI6InB1bHNhcjovLzEwLjIuMi4xMzk6NjY1MCJ9fSx7Im5hbWUiOiJjbGlja2hvdXNlIiwidmFsdWUiOnsidXJsIjoiamRiYzpjbGlja2hvdXNlOi8vMTAuMi4yLjI5OjgxMjMvaXNvcF9jZW50ZXI/dXNlX3RpbWVfem9uZT1Bc2lhL1NoYW5naGFpJnVzZV9zZXJ2ZXJfdGltZV96b25lPWZhbHNlIiwidXNlcm5hbWUiOiJkZWZhdWx0IiwicGFzc3dvcmQiOiIiLCJkcml2ZXItY2xhc3MtbmFtZSI6InJ1LnlhbmRleC5jbGlja2hvdXNlLkNsaWNrSG91c2VEcml2ZXIifX0seyJuYW1lIjoiaGRmcyIsInZhbHVlIjoiaGRmczovL2NsdXN0ZXItMTAxOjkwMDAvIn1dLCJydW5Db25maWciOnsiZXF1aXBJbmZvcyI6Ilt7XCJlcXVpcF9pZFwiOiBcIjFcIiwgXCJlcXVpcF90eXBlXCI6IFwiYnlxXCIsIFwic3RhdGlvbl9pZFwiOiBcImJkejAwMVwiLCBcImVxdWlwX25hbWVfZW5cIjogXCIxWkJcIn0sIHtcImVxdWlwX2lkXCI6IFwiMlwiLCBcImVxdWlwX3R5cGVcIjogXCJieXFcIiwgXCJzdGF0aW9uX2lkXCI6IFwiYmR6MDAxXCIsIFwiZXF1aXBfbmFtZV9lblwiOiBcIjJaQlwifV0iLCJydW5FbnYiOjEsInJ1bk1vZGVsIjoxLCJzY3JpcHRGaWxlTmFtZSI6ImhkZnM6Ly9jbHVzdGVyLTEwMTo5MDAwL3VzZXIvc29mdHdhcmUvY29tcHV0ZS1pbXBsLTEuMC1qYXItd2l0aC1kZXBlbmRlbmNpZXMuamFyIiwic2NyaXB0RnVuY3Rpb25OYW1lIjoiY29tLmNsb3Vkd2lzZS5pc29wLm1vZGVsLmltcGwuTXlDb21wdXRlIiwiZGF0YVN0YXJ0VGltZSI6MTYwNzk0MTA2OTk4NiwiZGF0YUVuZFRpbWUiOjE2MDc5NDEwNjk5ODYsImRhdGFIYW5kbGVUeXBlIjozLCJkYXRhSGFuZGxlVmFsdWUiOjE4MCwidGFza0lkIjoiMiJ9fQo=
报错信息
org.apache.flink.runtime.entrypoint.ClusterEntrypoint [] - Shutting YarnJobClusterEntrypoint down with application status FAILED. Diagnostics java.lang.AbstractMethodError: org.apache.flink.runtime.resourcemanager.ResourceManagerFactory.createResourceManager(Lorg/apache/flink/configuration/Configuration;Lorg/apache/flink/runtime/clusterframework/types/ResourceID;Lorg/apache/flink/runtime/rpc/RpcService;Lorg/apache/flink/runtime/highavailability/HighAvailabilityServices;Lorg/apache/flink/runtime/heartbeat/HeartbeatServices;Lorg/apache/flink/runtime/rpc/FatalErrorHandler;Lorg/apache/flink/runtime/entrypoint/ClusterInformation;Ljava/lang/String;Lorg/apache/flink/runtime/metrics/groups/ResourceManagerMetricGroup;Lorg/apache/flink/runtime/resourcemanager/ResourceManagerRuntimeServices;)Lorg/apache/flink/runtime/resourcemanager/ResourceManager;
at org.apache.flink.runtime.resourcemanager.ResourceManagerFactory.createResourceManager(ResourceManagerFactory.java:61)
at org.apache.flink.runtime.entrypoint.component.DefaultDispatcherResourceManagerComponentFactory.create(DefaultDispatcherResourceManagerComponentFactory.java:167)
at org.apache.flink.runtime.entrypoint.ClusterEntrypoint.runCluster(ClusterEntrypoint.java:216)
at org.apache.flink.runtime.entrypoint.ClusterEntrypoint.lambda$startCluster$0(ClusterEntrypoint.java:169)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1762)
at org.apache.flink.runtime.security.contexts.HadoopSecurityContext.runSecured(HadoopSecurityContext.java:41)
at org.apache.flink.runtime.entrypoint.ClusterEntrypoint.startCluster(ClusterEntrypoint.java:168)
at org.apache.flink.runtime.entrypoint.ClusterEntrypoint.runClusterEntrypoint(ClusterEntrypoint.java:517)
at org.apache.flink.yarn.entrypoint.YarnJobClusterEntrypoint.main(YarnJobClusterEntrypoint.java:95)
.
2020-12-23 10:20:00,282 INFO org.apache.flink.runtime.blob.BlobServer [] - Stopped BLOB server at 0.0.0.0:44886
2020-12-23 10:20:00,286 INFO org.apache.flink.runtime.rpc.akka.AkkaRpcService [] - Stopping Akka RPC service.
2020-12-23 10:20:00,299 INFO org.apache.flink.runtime.rpc.akka.AkkaRpcService [] - Stopping Akka RPC service.
2020-12-23 10:20:00,385 INFO akka.remote.RemoteActorRefProvider$RemotingTerminator [] - Shutting down remote daemon.
2020-12-23 10:20:00,389 INFO akka.remote.RemoteActorRefProvider$RemotingTerminator [] - Remote daemon shut down; proceeding with flushing remote transports.
2020-12-23 10:20:00,407 INFO akka.remote.RemoteActorRefProvider$RemotingTerminator [] - Shutting down remote daemon.
2020-12-23 10:20:00,408 INFO akka.remote.RemoteActorRefProvider$RemotingTerminator [] - Remote daemon shut down; proceeding with flushing remote transports.
2020-12-23 10:20:00,473 INFO akka.remote.RemoteActorRefProvider$RemotingTerminator [] - Remoting shut down.
2020-12-23 10:20:00,479 INFO akka.remote.RemoteActorRefProvider$RemotingTerminator [] - Remoting shut down.
2020-12-23 10:20:00,519 INFO org.apache.flink.runtime.rpc.akka.AkkaRpcService [] - Stopped Akka RPC service.
2020-12-23 10:20:00,519 INFO org.apache.flink.runtime.rpc.akka.AkkaRpcService [] - Stopped Akka RPC service.
2020-12-23 10:20:00,519 ERROR org.apache.flink.runtime.entrypoint.ClusterEntrypoint [] - Could not start cluster entrypoint YarnJobClusterEntrypoint.
org.apache.flink.runtime.entrypoint.ClusterEntrypointException: Failed to initialize the cluster entrypoint YarnJobClusterEntrypoint.
at org.apache.flink.runtime.entrypoint.ClusterEntrypoint.startCluster(ClusterEntrypoint.java:187) ~[model-compute-1.0.0-jar-with-dependencies.jar:?]
at org.apache.flink.runtime.entrypoint.ClusterEntrypoint.runClusterEntrypoint(ClusterEntrypoint.java:517) [model-compute-1.0.0-jar-with-dependencies.jar:?]
at org.apache.flink.yarn.entrypoint.YarnJobClusterEntrypoint.main(YarnJobClusterEntrypoint.java:95) [flink-dist_2.11-1.12.0.jar:1.12.0]
Caused by: java.lang.AbstractMethodError: org.apache.flink.runtime.resourcemanager.ResourceManagerFactory.createResourceManager(Lorg/apache/flink/configuration/Configuration;Lorg/apache/flink/runtime/clusterframework/types/ResourceID;Lorg/apache/flink/runtime/rpc/RpcService;Lorg/apache/flink/runtime/highavailability/HighAvailabilityServices;Lorg/apache/flink/runtime/heartbeat/HeartbeatServices;Lorg/apache/flink/runtime/rpc/FatalErrorHandler;Lorg/apache/flink/runtime/entrypoint/ClusterInformation;Ljava/lang/String;Lorg/apache/flink/runtime/metrics/groups/ResourceManagerMetricGroup;Lorg/apache/flink/runtime/resourcemanager/ResourceManagerRuntimeServices;)Lorg/apache/flink/runtime/resourcemanager/ResourceManager;
at org.apache.flink.runtime.resourcemanager.ResourceManagerFactory.createResourceManager(ResourceManagerFactory.java:61) ~[model-compute-1.0.0-jar-with-dependencies.jar:?]
at org.apache.flink.runtime.entrypoint.component.DefaultDispatcherResourceManagerComponentFactory.create(DefaultDispatcherResourceManagerComponentFactory.java:167) ~[model-compute-1.0.0-jar-with-dependencies.jar:?]
at org.apache.flink.runtime.entrypoint.ClusterEntrypoint.runCluster(ClusterEntrypoint.java:216) ~[model-compute-1.0.0-jar-with-dependencies.jar:?]
at org.apache.flink.runtime.entrypoint.ClusterEntrypoint.lambda$startCluster$0(ClusterEntrypoint.java:169) ~[model-compute-1.0.0-jar-with-dependencies.jar:?]
at java.security.AccessController.doPrivileged(Native Method) ~[?:1.8.0_201]
at javax.security.auth.Subject.doAs(Subject.java:422) ~[?:1.8.0_201]
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1762) ~[model-compute-1.0.0-jar-with-dependencies.jar:?]
at org.apache.flink.runtime.security.contexts.HadoopSecurityContext.runSecured(HadoopSecurityContext.java:41) ~[model-compute-1.0.0-jar-with-dependencies.jar:?]
at org.apache.flink.runtime.entrypoint.ClusterEntrypoint.startCluster(ClusterEntrypoint.java:168) ~[model-compute-1.0.0-jar-with-dependencies.jar:?]
... 2 more
解决方式,通过源码追踪找到:
org.apache.flink.yarn.entrypoint.YarnJobClusterEntrypoint.main
接着追踪到
org.apache.flink.runtime.entrypoint.ClusterEntrypoint.runCluster
org.apache.flink.runtime.entrypoint.component.DefaultDispatcherResourceManagerComponentFactory
发现 resourceManagerFactory 的实际实现类为:
YarnResourceManagerFactory
而 createResourceManager 的真正实现在
ActiveResourceManagerFactory
ResourceManagerFactory调用 createResourceManager()
在 flink 1.11.1 版本中 ActiveResourceManagerFactory 的 createResourceManager() 的抽象方法没有实现,而flink 1.12.0 是实现了的,在代码依赖中使用的是flink 1.11.1版本,导致次报错信息。
网友评论