gistfile1.txt

Job [1a10c097-5a10-4f19-a83d-769d59c12c19] submitted.
Waiting for job output...
16/12/13 00:44:16 INFO com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystemBase: GHFS version: 1.5.5-hadoop2
16/12/13 00:44:19 INFO org.spark_project.jetty.util.log: Logging initialized @3736ms
16/12/13 00:44:19 INFO org.spark_project.jetty.server.Server: jetty-9.2.z-SNAPSHOT
16/12/13 00:44:19 INFO org.spark_project.jetty.server.ServerConnector: Started ServerConnector@d7bffbc{HTTP/1.1}{0.0.0.0:4040}
16/12/13 00:44:19 INFO org.spark_project.jetty.server.Server: Started @3871ms
16/12/13 00:44:20 INFO org.apache.hadoop.yarn.client.RMProxy: Connecting to ResourceManager at venky-test-m/10.138.0.7:8032
16/12/13 00:44:20 ERROR org.apache.spark.SparkContext: Error initializing SparkContext.
java.io.FileNotFoundException: File file:/tmp/0afbad25-cb65-49f1-87b8-9cf6523512dd/skyfall-assembly-0.0.1.jar does not exist
        at org.apache.hadoop.fs.RawLocalFileSystem.deprecatedGetFileStatus(RawLocalFileSystem.java:611)
        at org.apache.hadoop.fs.RawLocalFileSystem.getFileLinkStatusInternal(RawLocalFileSystem.java:824)
        at org.apache.hadoop.fs.RawLocalFileSystem.getFileStatus(RawLocalFileSystem.java:601)
        at org.apache.hadoop.fs.FilterFileSystem.getFileStatus(FilterFileSystem.java:421)
        at org.apache.hadoop.fs.FileUtil.copy(FileUtil.java:337)
        at org.apache.hadoop.fs.FileUtil.copy(FileUtil.java:289)
        at org.apache.spark.deploy.yarn.Client.copyFileToRemote(Client.scala:340)
        at org.apache.spark.deploy.yarn.Client.org$apache$spark$deploy$yarn$Client$$distribute$1(Client.scala:433)
        at org.apache.spark.deploy.yarn.Client$$anonfun$prepareLocalResources$11$$anonfun$apply$6.apply(Client.scala:553)
        at org.apache.spark.deploy.yarn.Client$$anonfun$prepareLocalResources$11$$anonfun$apply$6.apply(Client.scala:552)
        at scala.collection.mutable.ArraySeq.foreach(ArraySeq.scala:74)
        at org.apache.spark.deploy.yarn.Client$$anonfun$prepareLocalResources$11.apply(Client.scala:552)
        at org.apache.spark.deploy.yarn.Client$$anonfun$prepareLocalResources$11.apply(Client.scala:551)
        at scala.collection.immutable.List.foreach(List.scala:381)
        at org.apache.spark.deploy.yarn.Client.prepareLocalResources(Client.scala:551)
        at org.apache.spark.deploy.yarn.Client.createContainerLaunchContext(Client.scala:834)
        at org.apache.spark.deploy.yarn.Client.submitApplication(Client.scala:167)
        at org.apache.spark.scheduler.cluster.YarnClientSchedulerBackend.start(YarnClientSchedulerBackend.scala:56)
        at org.apache.spark.scheduler.TaskSchedulerImpl.start(TaskSchedulerImpl.scala:149)
        at org.apache.spark.SparkContext.<init>(SparkContext.scala:497)
        at org.apache.spark.SparkContext$.getOrCreate(SparkContext.scala:2258)
        at org.apache.spark.streaming.StreamingContext.<init>(StreamingContext.scala:140)
        at org.apache.spark.streaming.StreamingContext$$anonfun$getOrCreate$1.apply(StreamingContext.scala:826)
        at org.apache.spark.streaming.StreamingContext$$anonfun$getOrCreate$1.apply(StreamingContext.scala:826)
        at scala.Option.map(Option.scala:146)
        at org.apache.spark.streaming.StreamingContext$.getOrCreate(StreamingContext.scala:826)
        at com.thumbtack.common.model.SparkStream$class.ssc(SparkStream.scala:34)
        at com.thumbtack.skyfall.Skyfall$.ssc$lzycompute(Skyfall.scala:37)
        at com.thumbtack.skyfall.Skyfall$.ssc(Skyfall.scala:37)
        at com.thumbtack.skyfall.Skyfall$.main(Skyfall.scala:154)
        at com.thumbtack.skyfall.Skyfall$.main(Skyfall.scala:37)
        at com.thumbtack.common.model.SparkStream$class.main(SparkStream.scala:58)
        at com.thumbtack.skyfall.Skyfall$.main(Skyfall.scala:37)
        at com.thumbtack.skyfall.Skyfall.main(Skyfall.scala)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:736)
        at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:185)
        at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:210)
        at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:124)
        at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
16/12/13 00:44:20 INFO org.spark_project.jetty.server.ServerConnector: Stopped ServerConnector@d7bffbc{HTTP/1.1}{0.0.0.0:4040}
16/12/13 00:44:20 WARN org.apache.spark.scheduler.cluster.YarnSchedulerBackend$YarnSchedulerEndpoint: Attempted to request executors before the AM has registered!
16/12/13 00:44:20 ERROR org.apache.spark.util.Utils: Uncaught exception in thread main
java.lang.NullPointerException
        at org.apache.spark.network.shuffle.ExternalShuffleClient.close(ExternalShuffleClient.java:152)
        at org.apache.spark.storage.BlockManager.stop(BlockManager.scala:1360)
        at org.apache.spark.SparkEnv.stop(SparkEnv.scala:87)
        at org.apache.spark.SparkContext$$anonfun$stop$11.apply$mcV$sp(SparkContext.scala:1797)
        at org.apache.spark.util.Utils$.tryLogNonFatalError(Utils.scala:1290)
        at org.apache.spark.SparkContext.stop(SparkContext.scala:1796)
        at org.apache.spark.SparkContext.<init>(SparkContext.scala:565)
        at org.apache.spark.SparkContext$.getOrCreate(SparkContext.scala:2258)
        at org.apache.spark.streaming.StreamingContext.<init>(StreamingContext.scala:140)
        at org.apache.spark.streaming.StreamingContext$$anonfun$getOrCreate$1.apply(StreamingContext.scala:826)
        at org.apache.spark.streaming.StreamingContext$$anonfun$getOrCreate$1.apply(StreamingContext.scala:826)
        at scala.Option.map(Option.scala:146)
        at org.apache.spark.streaming.StreamingContext$.getOrCreate(StreamingContext.scala:826)
        at com.thumbtack.common.model.SparkStream$class.ssc(SparkStream.scala:34)
        at com.thumbtack.skyfall.Skyfall$.ssc$lzycompute(Skyfall.scala:37)
        at com.thumbtack.skyfall.Skyfall$.ssc(Skyfall.scala:37)
        at com.thumbtack.skyfall.Skyfall$.main(Skyfall.scala:154)
        at com.thumbtack.skyfall.Skyfall$.main(Skyfall.scala:37)
        at com.thumbtack.common.model.SparkStream$class.main(SparkStream.scala:58)
        at com.thumbtack.skyfall.Skyfall$.main(Skyfall.scala:37)
        at com.thumbtack.skyfall.Skyfall.main(Skyfall.scala)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:736)
        at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:185)
        at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:210)
        at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:124)
        at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
Exception in thread "main" java.io.FileNotFoundException: File file:/tmp/0afbad25-cb65-49f1-87b8-9cf6523512dd/skyfall-assembly-0.0.1.jar does not exist
        at org.apache.hadoop.fs.RawLocalFileSystem.deprecatedGetFileStatus(RawLocalFileSystem.java:611)
        at org.apache.hadoop.fs.RawLocalFileSystem.getFileLinkStatusInternal(RawLocalFileSystem.java:824)
        at org.apache.hadoop.fs.RawLocalFileSystem.getFileStatus(RawLocalFileSystem.java:601)
        at org.apache.hadoop.fs.FilterFileSystem.getFileStatus(FilterFileSystem.java:421)
        at org.apache.hadoop.fs.FileUtil.copy(FileUtil.java:337)
        at org.apache.hadoop.fs.FileUtil.copy(FileUtil.java:289)
        at org.apache.spark.deploy.yarn.Client.copyFileToRemote(Client.scala:340)
        at org.apache.spark.deploy.yarn.Client.org$apache$spark$deploy$yarn$Client$$distribute$1(Client.scala:433)
        at org.apache.spark.deploy.yarn.Client$$anonfun$prepareLocalResources$11$$anonfun$apply$6.apply(Client.scala:553)
        at org.apache.spark.deploy.yarn.Client$$anonfun$prepareLocalResources$11$$anonfun$apply$6.apply(Client.scala:552)
        at scala.collection.mutable.ArraySeq.foreach(ArraySeq.scala:74)
        at org.apache.spark.deploy.yarn.Client$$anonfun$prepareLocalResources$11.apply(Client.scala:552)
        at org.apache.spark.deploy.yarn.Client$$anonfun$prepareLocalResources$11.apply(Client.scala:551)
        at scala.collection.immutable.List.foreach(List.scala:381)
        at org.apache.spark.deploy.yarn.Client.prepareLocalResources(Client.scala:551)
        at org.apache.spark.deploy.yarn.Client.createContainerLaunchContext(Client.scala:834)
        at org.apache.spark.deploy.yarn.Client.submitApplication(Client.scala:167)
        at org.apache.spark.scheduler.cluster.YarnClientSchedulerBackend.start(YarnClientSchedulerBackend.scala:56)
        at org.apache.spark.scheduler.TaskSchedulerImpl.start(TaskSchedulerImpl.scala:149)
        at org.apache.spark.SparkContext.<init>(SparkContext.scala:497)
        at org.apache.spark.SparkContext$.getOrCreate(SparkContext.scala:2258)
        at org.apache.spark.streaming.StreamingContext.<init>(StreamingContext.scala:140)
        at org.apache.spark.streaming.StreamingContext$$anonfun$getOrCreate$1.apply(StreamingContext.scala:826)
        at org.apache.spark.streaming.StreamingContext$$anonfun$getOrCreate$1.apply(StreamingContext.scala:826)
        at scala.Option.map(Option.scala:146)
        at org.apache.spark.streaming.StreamingContext$.getOrCreate(StreamingContext.scala:826)
        at com.thumbtack.common.model.SparkStream$class.ssc(SparkStream.scala:34)
        at com.thumbtack.skyfall.Skyfall$.ssc$lzycompute(Skyfall.scala:37)
        at com.thumbtack.skyfall.Skyfall$.ssc(Skyfall.scala:37)
        at com.thumbtack.skyfall.Skyfall$.main(Skyfall.scala:154)
        at com.thumbtack.skyfall.Skyfall$.main(Skyfall.scala:37)
        at com.thumbtack.common.model.SparkStream$class.main(SparkStream.scala:58)
        at com.thumbtack.skyfall.Skyfall$.main(Skyfall.scala:37)
        at com.thumbtack.skyfall.Skyfall.main(Skyfall.scala)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:736)
        at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:185)
        at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:210)
        at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:124)
        at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
ERROR: (gcloud.dataproc.jobs.submit.spark) Job [1a10c097-5a10-4f19-a83d-769d59c12c19] entered state [ERROR] while waiting for [DONE].