Skip to content

Instantly share code, notes, and snippets.

@soapergem

soapergem/app-history.log

Last active July 8, 2019 19:31
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save soapergem/6b379b5a9092dcd43777bdec8dee65a8 to your computer and use it in GitHub Desktop.
Save soapergem/6b379b5a9092dcd43777bdec8dee65a8 to your computer and use it in GitHub Desktop.
Download ZIP
EMR Exception Logs
Raw
app-history.log
User class threw exception: org.apache.spark.SparkException: Job aborted.
at org.apache.spark.internal.io.SparkHadoopWriter$.write(SparkHadoopWriter.scala:100)
at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1.apply$mcV$sp(PairRDDFunctions.scala:1096)
at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1.apply(PairRDDFunctions.scala:1094)
at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1.apply(PairRDDFunctions.scala:1094)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
at org.apache.spark.rdd.RDD.withScope(RDD.scala:363)
at org.apache.spark.rdd.PairRDDFunctions.saveAsHadoopDataset(PairRDDFunctions.scala:1094)
at com.mypackage.MyMainClassRunner$$anonfun$main$2.apply(MyMainClass.scala:385)
at com.mypackage.MyMainClassRunner$$anonfun$main$2.apply(MyMainClass.scala:350)
at scala.Option.map(Option.scala:146)
at com.mypackage.MyMainClassRunner$class.main(MyMainClass.scala:350)
at com.mypackage.MyMainClass$.main(MyMainClass.scala:398)
at com.mypackage.MyMainClass$.main(MyMainClass.scala:407)
at com.mypackage.MyMainClass.main(MyMainClass.scala)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.spark.deploy.yarn.ApplicationMaster$$anon$2.run(ApplicationMaster.scala:684)
Caused by: org.apache.spark.SparkException: Job aborted due to stage failure: Task 29 in stage 2.0 failed 4 times, most recent failure: Lost task 29.3 in stage 2.0 (TID 318, ip-10-23-1-194.ec2.internal, executor 3): org.apache.spark.SparkException: Task failed while writing rows
at org.apache.spark.internal.io.SparkHadoopWriter$.org$apache$spark$internal$io$SparkHadoopWriter$$executeTask(SparkHadoopWriter.scala:155)
at org.apache.spark.internal.io.SparkHadoopWriter$$anonfun$3.apply(SparkHadoopWriter.scala:83)
at org.apache.spark.internal.io.SparkHadoopWriter$$anonfun$3.apply(SparkHadoopWriter.scala:78)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
at org.apache.spark.scheduler.Task.run(Task.scala:121)
at org.apache.spark.executor.Executor$TaskRunner$$anonfun$10.apply(Executor.scala:408)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1360)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:414)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.RuntimeException: com.amazonaws.services.dynamodbv2.model.ResourceNotFoundException: Requested resource not found (Service: AmazonDynamoDBv2; Status Code: 400; Error Code: ResourceNotFoundException; Request ID: 0282QM8E8RTBMF3GBOG4IA3BJ7VV4KQNSO5AEMVJF66Q9ASUAAJG)
at org.apache.hadoop.dynamodb.DynamoDBFibonacciRetryer.handleException(DynamoDBFibonacciRetryer.java:108)
at org.apache.hadoop.dynamodb.DynamoDBFibonacciRetryer.runWithRetry(DynamoDBFibonacciRetryer.java:83)
at org.apache.hadoop.dynamodb.DynamoDBClient.writeBatch(DynamoDBClient.java:251)
at org.apache.hadoop.dynamodb.DynamoDBClient.putBatch(DynamoDBClient.java:208)
at org.apache.hadoop.dynamodb.write.AbstractDynamoDBRecordWriter.write(AbstractDynamoDBRecordWriter.java:112)
at org.apache.spark.internal.io.HadoopMapRedWriteConfigUtil.write(SparkHadoopWriter.scala:236)
at org.apache.spark.internal.io.SparkHadoopWriter$$anonfun$4.apply(SparkHadoopWriter.scala:130)
at org.apache.spark.internal.io.SparkHadoopWriter$$anonfun$4.apply(SparkHadoopWriter.scala:127)
at org.apache.spark.util.Utils$.tryWithSafeFinallyAndFailureCallbacks(Utils.scala:1394)
at org.apache.spark.internal.io.SparkHadoopWriter$.org$apache$spark$internal$io$SparkHadoopWriter$$executeTask(SparkHadoopWriter.scala:139)
... 10 more
Suppressed: java.lang.RuntimeException: com.amazonaws.services.dynamodbv2.model.ResourceNotFoundException: Requested resource not found (Service: AmazonDynamoDBv2; Status Code: 400; Error Code: ResourceNotFoundException; Request ID: 6VJ2NM3IM6OQJKP6I685DT6BHRVV4KQNSO5AEMVJF66Q9ASUAAJG)
at org.apache.hadoop.dynamodb.DynamoDBFibonacciRetryer.handleException(DynamoDBFibonacciRetryer.java:108)
at org.apache.hadoop.dynamodb.DynamoDBFibonacciRetryer.runWithRetry(DynamoDBFibonacciRetryer.java:83)
at org.apache.hadoop.dynamodb.DynamoDBClient.writeBatch(DynamoDBClient.java:251)
at org.apache.hadoop.dynamodb.DynamoDBClient.close(DynamoDBClient.java:235)
at org.apache.hadoop.dynamodb.write.AbstractDynamoDBRecordWriter.close(AbstractDynamoDBRecordWriter.java:137)
at org.apache.spark.internal.io.HadoopMapRedWriteConfigUtil.closeWriter(SparkHadoopWriter.scala:241)
at org.apache.spark.internal.io.SparkHadoopWriter$$anonfun$1.apply$mcV$sp(SparkHadoopWriter.scala:142)
at org.apache.spark.util.Utils$.tryWithSafeFinallyAndFailureCallbacks(Utils.scala:1403)
... 11 more
Caused by: com.amazonaws.services.dynamodbv2.model.ResourceNotFoundException: Requested resource not found (Service: AmazonDynamoDBv2; Status Code: 400; Error Code: ResourceNotFoundException; Request ID: 6VJ2NM3IM6OQJKP6I685DT6BHRVV4KQNSO5AEMVJF66Q9ASUAAJG)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1712)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1367)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1113)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:770)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:744)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:726)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:686)
at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:668)
at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:532)
at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:512)
at com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient.doInvoke(AmazonDynamoDBClient.java:4243)
at com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient.invoke(AmazonDynamoDBClient.java:4210)
at com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient.executeBatchWriteItem(AmazonDynamoDBClient.java:693)
at com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient.batchWriteItem(AmazonDynamoDBClient.java:660)
at org.apache.hadoop.dynamodb.DynamoDBClient$4.call(DynamoDBClient.java:258)
at org.apache.hadoop.dynamodb.DynamoDBClient$4.call(DynamoDBClient.java:252)
at org.apache.hadoop.dynamodb.DynamoDBFibonacciRetryer.runWithRetry(DynamoDBFibonacciRetryer.java:80)
... 17 more
Caused by: com.amazonaws.services.dynamodbv2.model.ResourceNotFoundException: Requested resource not found (Service: AmazonDynamoDBv2; Status Code: 400; Error Code: ResourceNotFoundException; Request ID: 0282QM8E8RTBMF3GBOG4IA3BJ7VV4KQNSO5AEMVJF66Q9ASUAAJG)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1712)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1367)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1113)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:770)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:744)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:726)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:686)
at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:668)
at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:532)
at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:512)
at com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient.doInvoke(AmazonDynamoDBClient.java:4243)
at com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient.invoke(AmazonDynamoDBClient.java:4210)
at com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient.executeBatchWriteItem(AmazonDynamoDBClient.java:693)
at com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient.batchWriteItem(AmazonDynamoDBClient.java:660)
at org.apache.hadoop.dynamodb.DynamoDBClient$4.call(DynamoDBClient.java:258)
at org.apache.hadoop.dynamodb.DynamoDBClient$4.call(DynamoDBClient.java:252)
at org.apache.hadoop.dynamodb.DynamoDBFibonacciRetryer.runWithRetry(DynamoDBFibonacciRetryer.java:80)
... 18 more
Driver stacktrace:
at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:2041)
at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:2029)
at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:2028)
at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48)
at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2028)
at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:966)
at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:966)
at scala.Option.foreach(Option.scala:257)
at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:966)
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2262)
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2211)
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2200)
at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49)
at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:777)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:2061)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:2082)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:2114)
at org.apache.spark.internal.io.SparkHadoopWriter$.write(SparkHadoopWriter.scala:78)
... 19 more
Caused by: org.apache.spark.SparkException: Task failed while writing rows
at org.apache.spark.internal.io.SparkHadoopWriter$.org$apache$spark$internal$io$SparkHadoopWriter$$executeTask(SparkHadoopWriter.scala:155)
at org.apache.spark.internal.io.SparkHadoopWriter$$anonfun$3.apply(SparkHadoopWriter.scala:83)
at org.apache.spark.internal.io.SparkHadoopWriter$$anonfun$3.apply(SparkHadoopWriter.scala:78)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
at org.apache.spark.scheduler.Task.run(Task.scala:121)
at org.apache.spark.executor.Executor$TaskRunner$$anonfun$10.apply(Executor.scala:408)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1360)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:414)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.RuntimeException: com.amazonaws.services.dynamodbv2.model.ResourceNotFoundException: Requested resource not found (Service: AmazonDynamoDBv2; Status Code: 400; Error Code: ResourceNotFoundException; Request ID: 0282QM8E8RTBMF3GBOG4IA3BJ7VV4KQNSO5AEMVJF66Q9ASUAAJG)
at org.apache.hadoop.dynamodb.DynamoDBFibonacciRetryer.handleException(DynamoDBFibonacciRetryer.java:108)
at org.apache.hadoop.dynamodb.DynamoDBFibonacciRetryer.runWithRetry(DynamoDBFibonacciRetryer.java:83)
at org.apache.hadoop.dynamodb.DynamoDBClient.writeBatch(DynamoDBClient.java:251)
at org.apache.hadoop.dynamodb.DynamoDBClient.putBatch(DynamoDBClient.java:208)
at org.apache.hadoop.dynamodb.write.AbstractDynamoDBRecordWriter.write(AbstractDynamoDBRecordWriter.java:112)
at org.apache.spark.internal.io.HadoopMapRedWriteConfigUtil.write(SparkHadoopWriter.scala:236)
at org.apache.spark.internal.io.SparkHadoopWriter$$anonfun$4.apply(SparkHadoopWriter.scala:130)
at org.apache.spark.internal.io.SparkHadoopWriter$$anonfun$4.apply(SparkHadoopWriter.scala:127)
at org.apache.spark.util.Utils$.tryWithSafeFinallyAndFailureCallbacks(Utils.scala:1394)
at org.apache.spark.internal.io.SparkHadoopWriter$.org$apache$spark$internal$io$SparkHadoopWriter$$executeTask(SparkHadoopWriter.scala:139)
... 10 more
Suppressed: java.lang.RuntimeException: com.amazonaws.services.dynamodbv2.model.ResourceNotFoundException: Requested resource not found (Service: AmazonDynamoDBv2; Status Code: 400; Error Code: ResourceNotFoundException; Request ID: 6VJ2NM3IM6OQJKP6I685DT6BHRVV4KQNSO5AEMVJF66Q9ASUAAJG)
at org.apache.hadoop.dynamodb.DynamoDBFibonacciRetryer.handleException(DynamoDBFibonacciRetryer.java:108)
at org.apache.hadoop.dynamodb.DynamoDBFibonacciRetryer.runWithRetry(DynamoDBFibonacciRetryer.java:83)
at org.apache.hadoop.dynamodb.DynamoDBClient.writeBatch(DynamoDBClient.java:251)
at org.apache.hadoop.dynamodb.DynamoDBClient.close(DynamoDBClient.java:235)
at org.apache.hadoop.dynamodb.write.AbstractDynamoDBRecordWriter.close(AbstractDynamoDBRecordWriter.java:137)
at org.apache.spark.internal.io.HadoopMapRedWriteConfigUtil.closeWriter(SparkHadoopWriter.scala:241)
at org.apache.spark.internal.io.SparkHadoopWriter$$anonfun$1.apply$mcV$sp(SparkHadoopWriter.scala:142)
at org.apache.spark.util.Utils$.tryWithSafeFinallyAndFailureCallbacks(Utils.scala:1403)
... 11 more
Caused by: com.amazonaws.services.dynamodbv2.model.ResourceNotFoundException: Requested resource not found (Service: AmazonDynamoDBv2; Status Code: 400; Error Code: ResourceNotFoundException; Request ID: 6VJ2NM3IM6OQJKP6I685DT6BHRVV4KQNSO5AEMVJF66Q9ASUAAJG)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1712)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1367)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1113)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:770)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:744)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:726)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:686)
at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:668)
at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:532)
at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:512)
at com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient.doInvoke(AmazonDynamoDBClient.java:4243)
at com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient.invoke(AmazonDynamoDBClient.java:4210)
at com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient.executeBatchWriteItem(AmazonDynamoDBClient.java:693)
at com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient.batchWriteItem(AmazonDynamoDBClient.java:660)
at org.apache.hadoop.dynamodb.DynamoDBClient$4.call(DynamoDBClient.java:258)
at org.apache.hadoop.dynamodb.DynamoDBClient$4.call(DynamoDBClient.java:252)
at org.apache.hadoop.dynamodb.DynamoDBFibonacciRetryer.runWithRetry(DynamoDBFibonacciRetryer.java:80)
... 17 more
Caused by: com.amazonaws.services.dynamodbv2.model.ResourceNotFoundException: Requested resource not found (Service: AmazonDynamoDBv2; Status Code: 400; Error Code: ResourceNotFoundException; Request ID: 0282QM8E8RTBMF3GBOG4IA3BJ7VV4KQNSO5AEMVJF66Q9ASUAAJG)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1712)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1367)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1113)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:770)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:744)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:726)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:686)
at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:668)
at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:532)
at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:512)
at com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient.doInvoke(AmazonDynamoDBClient.java:4243)
at com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient.invoke(AmazonDynamoDBClient.java:4210)
at com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient.executeBatchWriteItem(AmazonDynamoDBClient.java:693)
at com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient.batchWriteItem(AmazonDynamoDBClient.java:660)
at org.apache.hadoop.dynamodb.DynamoDBClient$4.call(DynamoDBClient.java:258)
at org.apache.hadoop.dynamodb.DynamoDBClient$4.call(DynamoDBClient.java:252)
at org.apache.hadoop.dynamodb.DynamoDBFibonacciRetryer.runWithRetry(DynamoDBFibonacciRetryer.java:80)
... 18 more
Raw
stderr.log
19/07/08 14:37:31 WARN DependencyUtils: Skip remote jar s3://my-bucket/my-spark-job.jar.
19/07/08 14:37:31 INFO RMProxy: Connecting to ResourceManager at ip-10-23-1-112.ec2.internal/10.23.1.112:8032
19/07/08 14:37:32 INFO Client: Requesting a new application from cluster with 4 NodeManagers
19/07/08 14:37:32 INFO Client: Verifying our application has not requested more than the maximum memory capability of the cluster (11520 MB per container)
19/07/08 14:37:32 INFO Client: Will allocate AM container, with 1408 MB memory including 384 MB overhead
19/07/08 14:37:32 INFO Client: Setting up container launch context for our AM
19/07/08 14:37:32 INFO Client: Setting up the launch environment for our AM container
19/07/08 14:37:32 INFO Client: Preparing resources for our AM container
19/07/08 14:37:32 WARN Client: Neither spark.yarn.jars nor spark.yarn.archive is set, falling back to uploading libraries under SPARK_HOME.
19/07/08 14:37:35 INFO Client: Uploading resource file:/mnt/tmp/spark-0a0d501b-2e74-46e2-8002-17588e7712c2/__spark_libs__6993199664096312703.zip -> hdfs://ip-10-23-1-112.ec2.internal:8020/user/hadoop/.sparkStaging/application_1562169635368_0004/__spark_libs__6993199664096312703.zip
19/07/08 14:37:38 INFO Client: Uploading resource s3://my-bucket/my-spark-job.jar -> hdfs://ip-10-23-1-112.ec2.internal:8020/user/hadoop/.sparkStaging/application_1562169635368_0004/my-spark-job.jar
19/07/08 14:37:38 INFO S3NativeFileSystem: Opening 's3://my-bucket/my-spark-job.jar' for reading
19/07/08 14:37:39 INFO Client: Uploading resource file:/etc/spark/conf/hive-site.xml -> hdfs://ip-10-23-1-112.ec2.internal:8020/user/hadoop/.sparkStaging/application_1562169635368_0004/hive-site.xml
19/07/08 14:37:39 INFO Client: Uploading resource file:/mnt/tmp/spark-0a0d501b-2e74-46e2-8002-17588e7712c2/__spark_conf__7349173317573360783.zip -> hdfs://ip-10-23-1-112.ec2.internal:8020/user/hadoop/.sparkStaging/application_1562169635368_0004/__spark_conf__.zip
19/07/08 14:37:39 INFO SecurityManager: Changing view acls to: hadoop
19/07/08 14:37:39 INFO SecurityManager: Changing modify acls to: hadoop
19/07/08 14:37:39 INFO SecurityManager: Changing view acls groups to:
19/07/08 14:37:39 INFO SecurityManager: Changing modify acls groups to:
19/07/08 14:37:39 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(hadoop); groups with view permissions: Set(); users with modify permissions: Set(hadoop); groups with modify permissions: Set()
19/07/08 14:37:41 INFO Client: Submitting application application_1562169635368_0004 to ResourceManager
19/07/08 14:37:41 INFO YarnClientImpl: Submitted application application_1562169635368_0004
19/07/08 14:37:42 INFO Client: Application report for application_1562169635368_0004 (state: ACCEPTED)
19/07/08 14:37:42 INFO Client:
client token: N/A
diagnostics: AM container is launched, waiting for AM container to Register with RM
ApplicationMaster host: N/A
ApplicationMaster RPC port: -1
queue: default
start time: 1562596661483
final status: UNDEFINED
tracking URL: http://ip-10-23-1-112.ec2.internal:20888/proxy/application_1562169635368_0004/
user: hadoop
19/07/08 14:37:43 INFO Client: Application report for application_1562169635368_0004 (state: ACCEPTED)
19/07/08 14:37:44 INFO Client: Application report for application_1562169635368_0004 (state: ACCEPTED)
19/07/08 14:37:45 INFO Client: Application report for application_1562169635368_0004 (state: ACCEPTED)
19/07/08 14:37:46 INFO Client: Application report for application_1562169635368_0004 (state: ACCEPTED)
19/07/08 14:37:47 INFO Client: Application report for application_1562169635368_0004 (state: ACCEPTED)
19/07/08 14:37:48 INFO Client: Application report for application_1562169635368_0004 (state: ACCEPTED)
19/07/08 14:37:49 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:37:49 INFO Client:
client token: N/A
diagnostics: N/A
ApplicationMaster host: ip-10-23-1-194.ec2.internal
ApplicationMaster RPC port: 35671
queue: default
start time: 1562596661483
final status: UNDEFINED
tracking URL: http://ip-10-23-1-112.ec2.internal:20888/proxy/application_1562169635368_0004/
user: hadoop
19/07/08 14:37:50 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:37:51 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:37:52 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:37:53 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:37:54 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:37:55 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:37:56 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:37:57 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:37:58 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:37:59 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:38:00 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:38:01 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:38:02 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:38:03 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:38:04 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:38:05 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:38:06 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:38:07 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:38:08 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:38:09 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:38:10 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:38:11 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:38:12 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:38:13 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:38:14 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:38:15 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:38:16 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:38:17 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:38:18 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:38:19 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:38:20 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:38:21 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:38:22 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:38:23 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:38:24 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:38:25 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:38:26 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:38:27 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:38:28 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:38:29 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:38:30 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:38:31 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:38:32 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:38:33 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:38:34 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:38:35 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:38:36 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:38:37 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:38:38 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:38:39 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:38:40 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:38:41 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:38:42 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:38:43 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:38:44 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:38:45 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:38:46 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:38:47 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:38:48 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:38:49 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:38:50 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:38:51 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:38:52 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:38:53 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:38:54 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:38:55 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:38:56 INFO Client: Application report for application_1562169635368_0004 (state: ACCEPTED)
19/07/08 14:38:56 INFO Client:
client token: N/A
diagnostics: AM container is launched, waiting for AM container to Register with RM
ApplicationMaster host: N/A
ApplicationMaster RPC port: -1
queue: default
start time: 1562596661483
final status: UNDEFINED
tracking URL: http://ip-10-23-1-112.ec2.internal:20888/proxy/application_1562169635368_0004/
user: hadoop
19/07/08 14:38:57 INFO Client: Application report for application_1562169635368_0004 (state: ACCEPTED)
19/07/08 14:38:58 INFO Client: Application report for application_1562169635368_0004 (state: ACCEPTED)
19/07/08 14:38:59 INFO Client: Application report for application_1562169635368_0004 (state: ACCEPTED)
19/07/08 14:39:00 INFO Client: Application report for application_1562169635368_0004 (state: ACCEPTED)
19/07/08 14:39:01 INFO Client: Application report for application_1562169635368_0004 (state: ACCEPTED)
19/07/08 14:39:02 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:39:02 INFO Client:
client token: N/A
diagnostics: N/A
ApplicationMaster host: ip-10-23-1-52.ec2.internal
ApplicationMaster RPC port: 42005
queue: default
start time: 1562596661483
final status: UNDEFINED
tracking URL: http://ip-10-23-1-112.ec2.internal:20888/proxy/application_1562169635368_0004/
user: hadoop
19/07/08 14:39:03 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:39:04 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:39:05 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:39:06 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:39:07 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:39:08 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:39:09 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:39:10 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:39:11 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:39:12 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:39:13 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:39:14 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:39:15 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:39:16 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:39:17 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:39:18 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:39:19 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:39:20 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:39:21 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:39:22 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:39:23 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:39:24 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:39:25 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:39:26 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:39:27 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:39:28 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:39:29 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:39:30 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:39:31 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:39:32 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:39:33 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:39:34 INFO Client: Application report for application_1562169635368_0004 (state: RUNNING)
19/07/08 14:39:35 INFO Client: Application report for application_1562169635368_0004 (state: FINISHED)
19/07/08 14:39:35 INFO Client:
client token: N/A
diagnostics: User class threw exception: org.apache.spark.SparkException: Job aborted.
at org.apache.spark.internal.io.SparkHadoopWriter$.write(SparkHadoopWriter.scala:100)
at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1.apply$mcV$sp(PairRDDFunctions.scala:1096)
at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1.apply(PairRDDFunctions.scala:1094)
at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1.apply(PairRDDFunctions.scala:1094)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
at org.apache.spark.rdd.RDD.withScope(RDD.scala:363)
at org.apache.spark.rdd.PairRDDFunctions.saveAsHadoopDataset(PairRDDFunctions.scala:1094)
at com.mypackage.MyMainClassRunner$$anonfun$main$2.apply(MyMainClass.scala:385)
at com.mypackage.MyMainClassRunner$$anonfun$main$2.apply(MyMainClass.scala:350)
at scala.Option.map(Option.scala:146)
at com.mypackage.MyMainClassRunner$class.main(MyMainClass.scala:350)
at com.mypackage.MyMainClass$.main(MyMainClass.scala:398)
at com.mypackage.MyMainClass$.main(MyMainClass.scala:407)
at com.mypackage.MyMainClass.main(MyMainClass.scala)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.spark.deploy.yarn.ApplicationMaster$$anon$2.run(ApplicationMaster.scala:684)
Caused by: org.apache.spark.SparkException: Job aborted due to stage failure: Task 4 in stage 2.0 failed 4 times, most recent failure: Lost task 4.3 in stage 2.0 (TID 298, ip-10-23-1-52.ec2.internal, executor 3): java.lang.RuntimeException: Could not lookup table EmrTest_20190708143902 in DynamoDB.
at org.apache.hadoop.dynamodb.DynamoDBClient.describeTable(DynamoDBClient.java:136)
at org.apache.hadoop.dynamodb.write.WriteIopsCalculator.getThroughput(WriteIopsCalculator.java:86)
at org.apache.hadoop.dynamodb.write.WriteIopsCalculator.calculateTargetIops(WriteIopsCalculator.java:68)
at org.apache.hadoop.dynamodb.IopsController.<init>(IopsController.java:40)
at org.apache.hadoop.dynamodb.write.AbstractDynamoDBRecordWriter.<init>(AbstractDynamoDBRecordWriter.java:87)
at org.apache.hadoop.dynamodb.write.DefaultDynamoDBRecordWriter.<init>(DefaultDynamoDBRecordWriter.java:27)
at org.apache.hadoop.dynamodb.write.DynamoDBOutputFormat.getRecordWriter(DynamoDBOutputFormat.java:30)
at org.apache.spark.internal.io.HadoopMapRedWriteConfigUtil.initWriter(SparkHadoopWriter.scala:228)
at org.apache.spark.internal.io.SparkHadoopWriter$.org$apache$spark$internal$io$SparkHadoopWriter$$executeTask(SparkHadoopWriter.scala:122)
at org.apache.spark.internal.io.SparkHadoopWriter$$anonfun$3.apply(SparkHadoopWriter.scala:83)
at org.apache.spark.internal.io.SparkHadoopWriter$$anonfun$3.apply(SparkHadoopWriter.scala:78)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
at org.apache.spark.scheduler.Task.run(Task.scala:121)
at org.apache.spark.executor.Executor$TaskRunner$$anonfun$10.apply(Executor.scala:408)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1360)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:414)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.RuntimeException: com.amazonaws.services.dynamodbv2.model.ResourceNotFoundException: Requested resource not found: Table: EmrTest_20190708143902 not found (Service: AmazonDynamoDBv2; Status Code: 400; Error Code: ResourceNotFoundException; Request ID: V0M91J7KEUVR4VM78MF5TKHLEBVV4KQNSO5AEMVJF66Q9ASUAAJG)
at org.apache.hadoop.dynamodb.DynamoDBFibonacciRetryer.handleException(DynamoDBFibonacciRetryer.java:108)
at org.apache.hadoop.dynamodb.DynamoDBFibonacciRetryer.runWithRetry(DynamoDBFibonacciRetryer.java:83)
at org.apache.hadoop.dynamodb.DynamoDBClient.describeTable(DynamoDBClient.java:125)
... 18 more
Caused by: com.amazonaws.services.dynamodbv2.model.ResourceNotFoundException: Requested resource not found: Table: EmrTest_20190708143902 not found (Service: AmazonDynamoDBv2; Status Code: 400; Error Code: ResourceNotFoundException; Request ID: V0M91J7KEUVR4VM78MF5TKHLEBVV4KQNSO5AEMVJF66Q9ASUAAJG)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1712)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1367)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1113)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:770)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:744)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:726)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:686)
at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:668)
at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:532)
at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:512)
at com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient.doInvoke(AmazonDynamoDBClient.java:4243)
at com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient.invoke(AmazonDynamoDBClient.java:4210)
at com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient.executeDescribeTable(AmazonDynamoDBClient.java:1890)
at com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient.describeTable(AmazonDynamoDBClient.java:1857)
at org.apache.hadoop.dynamodb.DynamoDBClient$1.call(DynamoDBClient.java:129)
at org.apache.hadoop.dynamodb.DynamoDBClient$1.call(DynamoDBClient.java:126)
at org.apache.hadoop.dynamodb.DynamoDBFibonacciRetryer.runWithRetry(DynamoDBFibonacciRetryer.java:80)
... 19 more
Driver stacktrace:
at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:2041)
at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:2029)
at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:2028)
at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48)
at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2028)
at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:966)
at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:966)
at scala.Option.foreach(Option.scala:257)
at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:966)
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2262)
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2211)
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2200)
at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49)
at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:777)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:2061)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:2082)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:2114)
at org.apache.spark.internal.io.SparkHadoopWriter$.write(SparkHadoopWriter.scala:78)
... 19 more
Caused by: java.lang.RuntimeException: Could not lookup table EmrTest_20190708143902 in DynamoDB.
at org.apache.hadoop.dynamodb.DynamoDBClient.describeTable(DynamoDBClient.java:136)
at org.apache.hadoop.dynamodb.write.WriteIopsCalculator.getThroughput(WriteIopsCalculator.java:86)
at org.apache.hadoop.dynamodb.write.WriteIopsCalculator.calculateTargetIops(WriteIopsCalculator.java:68)
at org.apache.hadoop.dynamodb.IopsController.<init>(IopsController.java:40)
at org.apache.hadoop.dynamodb.write.AbstractDynamoDBRecordWriter.<init>(AbstractDynamoDBRecordWriter.java:87)
at org.apache.hadoop.dynamodb.write.DefaultDynamoDBRecordWriter.<init>(DefaultDynamoDBRecordWriter.java:27)
at org.apache.hadoop.dynamodb.write.DynamoDBOutputFormat.getRecordWriter(DynamoDBOutputFormat.java:30)
at org.apache.spark.internal.io.HadoopMapRedWriteConfigUtil.initWriter(SparkHadoopWriter.scala:228)
at org.apache.spark.internal.io.SparkHadoopWriter$.org$apache$spark$internal$io$SparkHadoopWriter$$executeTask(SparkHadoopWriter.scala:122)
at org.apache.spark.internal.io.SparkHadoopWriter$$anonfun$3.apply(SparkHadoopWriter.scala:83)
at org.apache.spark.internal.io.SparkHadoopWriter$$anonfun$3.apply(SparkHadoopWriter.scala:78)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
at org.apache.spark.scheduler.Task.run(Task.scala:121)
at org.apache.spark.executor.Executor$TaskRunner$$anonfun$10.apply(Executor.scala:408)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1360)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:414)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.RuntimeException: com.amazonaws.services.dynamodbv2.model.ResourceNotFoundException: Requested resource not found: Table: EmrTest_20190708143902 not found (Service: AmazonDynamoDBv2; Status Code: 400; Error Code: ResourceNotFoundException; Request ID: V0M91J7KEUVR4VM78MF5TKHLEBVV4KQNSO5AEMVJF66Q9ASUAAJG)
at org.apache.hadoop.dynamodb.DynamoDBFibonacciRetryer.handleException(DynamoDBFibonacciRetryer.java:108)
at org.apache.hadoop.dynamodb.DynamoDBFibonacciRetryer.runWithRetry(DynamoDBFibonacciRetryer.java:83)
at org.apache.hadoop.dynamodb.DynamoDBClient.describeTable(DynamoDBClient.java:125)
... 18 more
Caused by: com.amazonaws.services.dynamodbv2.model.ResourceNotFoundException: Requested resource not found: Table: EmrTest_20190708143902 not found (Service: AmazonDynamoDBv2; Status Code: 400; Error Code: ResourceNotFoundException; Request ID: V0M91J7KEUVR4VM78MF5TKHLEBVV4KQNSO5AEMVJF66Q9ASUAAJG)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1712)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1367)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1113)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:770)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:744)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:726)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:686)
at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:668)
at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:532)
at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:512)
at com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient.doInvoke(AmazonDynamoDBClient.java:4243)
at com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient.invoke(AmazonDynamoDBClient.java:4210)
at com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient.executeDescribeTable(AmazonDynamoDBClient.java:1890)
at com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient.describeTable(AmazonDynamoDBClient.java:1857)
at org.apache.hadoop.dynamodb.DynamoDBClient$1.call(DynamoDBClient.java:129)
at org.apache.hadoop.dynamodb.DynamoDBClient$1.call(DynamoDBClient.java:126)
at org.apache.hadoop.dynamodb.DynamoDBFibonacciRetryer.runWithRetry(DynamoDBFibonacciRetryer.java:80)
... 19 more
ApplicationMaster host: ip-10-23-1-52.ec2.internal
ApplicationMaster RPC port: 42005
queue: default
start time: 1562596661483
final status: FAILED
tracking URL: http://ip-10-23-1-112.ec2.internal:20888/proxy/application_1562169635368_0004/
user: hadoop
19/07/08 14:39:35 ERROR Client: Application diagnostics message: User class threw exception: org.apache.spark.SparkException: Job aborted.
at org.apache.spark.internal.io.SparkHadoopWriter$.write(SparkHadoopWriter.scala:100)
at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1.apply$mcV$sp(PairRDDFunctions.scala:1096)
at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1.apply(PairRDDFunctions.scala:1094)
at org.apache.spark.rdd.PairRDDFunctions$$anonfun$saveAsHadoopDataset$1.apply(PairRDDFunctions.scala:1094)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
at org.apache.spark.rdd.RDD.withScope(RDD.scala:363)
at org.apache.spark.rdd.PairRDDFunctions.saveAsHadoopDataset(PairRDDFunctions.scala:1094)
at com.mypackage.MyMainClassRunner$$anonfun$main$2.apply(MyMainClass.scala:385)
at com.mypackage.MyMainClassRunner$$anonfun$main$2.apply(MyMainClass.scala:350)
at scala.Option.map(Option.scala:146)
at com.mypackage.MyMainClassRunner$class.main(MyMainClass.scala:350)
at com.mypackage.MyMainClass$.main(MyMainClass.scala:398)
at com.mypackage.MyMainClass$.main(MyMainClass.scala:407)
at com.mypackage.MyMainClass.main(MyMainClass.scala)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.spark.deploy.yarn.ApplicationMaster$$anon$2.run(ApplicationMaster.scala:684)
Caused by: org.apache.spark.SparkException: Job aborted due to stage failure: Task 4 in stage 2.0 failed 4 times, most recent failure: Lost task 4.3 in stage 2.0 (TID 298, ip-10-23-1-52.ec2.internal, executor 3): java.lang.RuntimeException: Could not lookup table EmrTest_20190708143902 in DynamoDB.
at org.apache.hadoop.dynamodb.DynamoDBClient.describeTable(DynamoDBClient.java:136)
at org.apache.hadoop.dynamodb.write.WriteIopsCalculator.getThroughput(WriteIopsCalculator.java:86)
at org.apache.hadoop.dynamodb.write.WriteIopsCalculator.calculateTargetIops(WriteIopsCalculator.java:68)
at org.apache.hadoop.dynamodb.IopsController.<init>(IopsController.java:40)
at org.apache.hadoop.dynamodb.write.AbstractDynamoDBRecordWriter.<init>(AbstractDynamoDBRecordWriter.java:87)
at org.apache.hadoop.dynamodb.write.DefaultDynamoDBRecordWriter.<init>(DefaultDynamoDBRecordWriter.java:27)
at org.apache.hadoop.dynamodb.write.DynamoDBOutputFormat.getRecordWriter(DynamoDBOutputFormat.java:30)
at org.apache.spark.internal.io.HadoopMapRedWriteConfigUtil.initWriter(SparkHadoopWriter.scala:228)
at org.apache.spark.internal.io.SparkHadoopWriter$.org$apache$spark$internal$io$SparkHadoopWriter$$executeTask(SparkHadoopWriter.scala:122)
at org.apache.spark.internal.io.SparkHadoopWriter$$anonfun$3.apply(SparkHadoopWriter.scala:83)
at org.apache.spark.internal.io.SparkHadoopWriter$$anonfun$3.apply(SparkHadoopWriter.scala:78)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
at org.apache.spark.scheduler.Task.run(Task.scala:121)
at org.apache.spark.executor.Executor$TaskRunner$$anonfun$10.apply(Executor.scala:408)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1360)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:414)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.RuntimeException: com.amazonaws.services.dynamodbv2.model.ResourceNotFoundException: Requested resource not found: Table: EmrTest_20190708143902 not found (Service: AmazonDynamoDBv2; Status Code: 400; Error Code: ResourceNotFoundException; Request ID: V0M91J7KEUVR4VM78MF5TKHLEBVV4KQNSO5AEMVJF66Q9ASUAAJG)
at org.apache.hadoop.dynamodb.DynamoDBFibonacciRetryer.handleException(DynamoDBFibonacciRetryer.java:108)
at org.apache.hadoop.dynamodb.DynamoDBFibonacciRetryer.runWithRetry(DynamoDBFibonacciRetryer.java:83)
at org.apache.hadoop.dynamodb.DynamoDBClient.describeTable(DynamoDBClient.java:125)
... 18 more
Caused by: com.amazonaws.services.dynamodbv2.model.ResourceNotFoundException: Requested resource not found: Table: EmrTest_20190708143902 not found (Service: AmazonDynamoDBv2; Status Code: 400; Error Code: ResourceNotFoundException; Request ID: V0M91J7KEUVR4VM78MF5TKHLEBVV4KQNSO5AEMVJF66Q9ASUAAJG)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1712)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1367)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1113)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:770)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:744)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:726)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:686)
at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:668)
at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:532)
at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:512)
at com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient.doInvoke(AmazonDynamoDBClient.java:4243)
at com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient.invoke(AmazonDynamoDBClient.java:4210)
at com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient.executeDescribeTable(AmazonDynamoDBClient.java:1890)
at com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient.describeTable(AmazonDynamoDBClient.java:1857)
at org.apache.hadoop.dynamodb.DynamoDBClient$1.call(DynamoDBClient.java:129)
at org.apache.hadoop.dynamodb.DynamoDBClient$1.call(DynamoDBClient.java:126)
at org.apache.hadoop.dynamodb.DynamoDBFibonacciRetryer.runWithRetry(DynamoDBFibonacciRetryer.java:80)
... 19 more
Driver stacktrace:
at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:2041)
at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:2029)
at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:2028)
at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48)
at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2028)
at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:966)
at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:966)
at scala.Option.foreach(Option.scala:257)
at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:966)
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2262)
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2211)
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2200)
at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49)
at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:777)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:2061)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:2082)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:2114)
at org.apache.spark.internal.io.SparkHadoopWriter$.write(SparkHadoopWriter.scala:78)
... 19 more
Caused by: java.lang.RuntimeException: Could not lookup table EmrTest_20190708143902 in DynamoDB.
at org.apache.hadoop.dynamodb.DynamoDBClient.describeTable(DynamoDBClient.java:136)
at org.apache.hadoop.dynamodb.write.WriteIopsCalculator.getThroughput(WriteIopsCalculator.java:86)
at org.apache.hadoop.dynamodb.write.WriteIopsCalculator.calculateTargetIops(WriteIopsCalculator.java:68)
at org.apache.hadoop.dynamodb.IopsController.<init>(IopsController.java:40)
at org.apache.hadoop.dynamodb.write.AbstractDynamoDBRecordWriter.<init>(AbstractDynamoDBRecordWriter.java:87)
at org.apache.hadoop.dynamodb.write.DefaultDynamoDBRecordWriter.<init>(DefaultDynamoDBRecordWriter.java:27)
at org.apache.hadoop.dynamodb.write.DynamoDBOutputFormat.getRecordWriter(DynamoDBOutputFormat.java:30)
at org.apache.spark.internal.io.HadoopMapRedWriteConfigUtil.initWriter(SparkHadoopWriter.scala:228)
at org.apache.spark.internal.io.SparkHadoopWriter$.org$apache$spark$internal$io$SparkHadoopWriter$$executeTask(SparkHadoopWriter.scala:122)
at org.apache.spark.internal.io.SparkHadoopWriter$$anonfun$3.apply(SparkHadoopWriter.scala:83)
at org.apache.spark.internal.io.SparkHadoopWriter$$anonfun$3.apply(SparkHadoopWriter.scala:78)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
at org.apache.spark.scheduler.Task.run(Task.scala:121)
at org.apache.spark.executor.Executor$TaskRunner$$anonfun$10.apply(Executor.scala:408)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1360)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:414)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.RuntimeException: com.amazonaws.services.dynamodbv2.model.ResourceNotFoundException: Requested resource not found: Table: EmrTest_20190708143902 not found (Service: AmazonDynamoDBv2; Status Code: 400; Error Code: ResourceNotFoundException; Request ID: V0M91J7KEUVR4VM78MF5TKHLEBVV4KQNSO5AEMVJF66Q9ASUAAJG)
at org.apache.hadoop.dynamodb.DynamoDBFibonacciRetryer.handleException(DynamoDBFibonacciRetryer.java:108)
at org.apache.hadoop.dynamodb.DynamoDBFibonacciRetryer.runWithRetry(DynamoDBFibonacciRetryer.java:83)
at org.apache.hadoop.dynamodb.DynamoDBClient.describeTable(DynamoDBClient.java:125)
... 18 more
Caused by: com.amazonaws.services.dynamodbv2.model.ResourceNotFoundException: Requested resource not found: Table: EmrTest_20190708143902 not found (Service: AmazonDynamoDBv2; Status Code: 400; Error Code: ResourceNotFoundException; Request ID: V0M91J7KEUVR4VM78MF5TKHLEBVV4KQNSO5AEMVJF66Q9ASUAAJG)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1712)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1367)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1113)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:770)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:744)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:726)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:686)
at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:668)
at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:532)
at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:512)
at com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient.doInvoke(AmazonDynamoDBClient.java:4243)
at com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient.invoke(AmazonDynamoDBClient.java:4210)
at com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient.executeDescribeTable(AmazonDynamoDBClient.java:1890)
at com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient.describeTable(AmazonDynamoDBClient.java:1857)
at org.apache.hadoop.dynamodb.DynamoDBClient$1.call(DynamoDBClient.java:129)
at org.apache.hadoop.dynamodb.DynamoDBClient$1.call(DynamoDBClient.java:126)
at org.apache.hadoop.dynamodb.DynamoDBFibonacciRetryer.runWithRetry(DynamoDBFibonacciRetryer.java:80)
... 19 more
Exception in thread "main" org.apache.spark.SparkException: Application application_1562169635368_0004 finished with failed status
at org.apache.spark.deploy.yarn.Client.run(Client.scala:1148)
at org.apache.spark.deploy.yarn.YarnClusterApplication.start(Client.scala:1525)
at org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:849)
at org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:167)
at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:195)
at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:86)
at org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:924)
at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:933)
at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
19/07/08 14:39:35 INFO ShutdownHookManager: Shutdown hook called
19/07/08 14:39:35 INFO ShutdownHookManager: Deleting directory /mnt/tmp/spark-0a0d501b-2e74-46e2-8002-17588e7712c2
19/07/08 14:39:35 INFO ShutdownHookManager: Deleting directory /mnt/tmp/spark-9d5cf9da-a730-4a60-81dc-1ac5e2a3cd80
Command exiting with ret '1'
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment