Created
December 13, 2018 05:36
-
-
Save rohitgcs/17940e4804b15aa0855b420539a86805 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| 18/12/13 05:32:50 INFO org.elasticsearch.hadoop.util.Version: Elasticsearch Hadoop v5.6.9 [491e9397f3] | |
| 18/12/13 05:32:51 WARN org.apache.spark.scheduler.TaskSetManager: Lost task 1.0 in stage 4.0 (TID 8, cluster-dcd-w-1.c.datascience-214900.internal, executor 1): java.lang.ClassNotFoundException: org.elasticsearch.spark.sql.EsSparkSQL$$anonfun$saveToEs$1 | |
| at java.net.URLClassLoader.findClass(URLClassLoader.java:381) | |
| at java.lang.ClassLoader.loadClass(ClassLoader.java:424) | |
| at java.lang.ClassLoader.loadClass(ClassLoader.java:357) | |
| at java.lang.Class.forName0(Native Method) | |
| at java.lang.Class.forName(Class.java:348) | |
| at org.apache.spark.serializer.JavaDeserializationStream$$anon$1.resolveClass(JavaSerializer.scala:67) | |
| at java.io.ObjectInputStream.readNonProxyDesc(ObjectInputStream.java:1866) | |
| at java.io.ObjectInputStream.readClassDesc(ObjectInputStream.java:1749) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2040) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1571) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:2285) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2209) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2067) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1571) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:431) | |
| at org.apache.spark.serializer.JavaDeserializationStream.readObject(JavaSerializer.scala:75) | |
| at org.apache.spark.serializer.JavaSerializerInstance.deserialize(JavaSerializer.scala:114) | |
| at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:80) | |
| at org.apache.spark.scheduler.Task.run(Task.scala:108) | |
| at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:338) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| 18/12/13 05:32:51 ERROR org.apache.spark.scheduler.TaskSetManager: Task 0 in stage 4.0 failed 4 times; aborting job | |
| 18/12/13 05:32:51 WARN org.apache.spark.ExecutorAllocationManager: No stages are running, but numRunningTasks != 0 | |
| Traceback (most recent call last): | |
| File "/home/Rohit/pyscripts/adhoc/adhocScripts/ClickToDeliver/ShipToDeliver/saveDataElastic.py", line 216, in <module> | |
| .write.options(**es_write_conf).mode("append").format("org.elasticsearch.spark.sql").save(esPath) | |
| File "/usr/lib/spark/python/lib/pyspark.zip/pyspark/sql/readwriter.py", line 595, in save | |
| File "/usr/lib/spark/python/lib/py4j-0.10.4-src.zip/py4j/java_gateway.py", line 1133, in __call__ | |
| File "/usr/lib/spark/python/lib/pyspark.zip/pyspark/sql/utils.py", line 63, in deco | |
| File "/usr/lib/spark/python/lib/py4j-0.10.4-src.zip/py4j/protocol.py", line 319, in get_return_value | |
| py4j.protocol.Py4JJavaError: An error occurred while calling o204.save. | |
| : org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 4.0 failed 4 times, most recent failure: Lost task 0.3 in stage 4.0 (TID 16, cluster-dcd-w-1.c.datascience-214900.internal, executor 1): java.lang.ClassNotFoundException: org.elasticsearch.spark.sql.EsSparkSQL$$anonfun$saveToEs$1 | |
| at java.net.URLClassLoader.findClass(URLClassLoader.java:381) | |
| at java.lang.ClassLoader.loadClass(ClassLoader.java:424) | |
| at java.lang.ClassLoader.loadClass(ClassLoader.java:357) | |
| at java.lang.Class.forName0(Native Method) | |
| at java.lang.Class.forName(Class.java:348) | |
| at org.apache.spark.serializer.JavaDeserializationStream$$anon$1.resolveClass(JavaSerializer.scala:67) | |
| at java.io.ObjectInputStream.readNonProxyDesc(ObjectInputStream.java:1866) | |
| at java.io.ObjectInputStream.readClassDesc(ObjectInputStream.java:1749) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2040) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1571) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:2285) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2209) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2067) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1571) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:431) | |
| at org.apache.spark.serializer.JavaDeserializationStream.readObject(JavaSerializer.scala:75) | |
| at org.apache.spark.serializer.JavaSerializerInstance.deserialize(JavaSerializer.scala:114) | |
| at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:80) | |
| at org.apache.spark.scheduler.Task.run(Task.scala:108) | |
| at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:338) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Driver stacktrace: | |
| at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1517) | |
| at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1505) | |
| at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1504) | |
| at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) | |
| at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) | |
| at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1504) | |
| at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:814) | |
| at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:814) | |
| at scala.Option.foreach(Option.scala:257) | |
| at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:814) | |
| at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:1732) | |
| at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1687) | |
| at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1676) | |
| at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48) | |
| at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:630) | |
| at org.apache.spark.SparkContext.runJob(SparkContext.scala:2029) | |
| at org.apache.spark.SparkContext.runJob(SparkContext.scala:2050) | |
| at org.apache.spark.SparkContext.runJob(SparkContext.scala:2082) | |
| at org.elasticsearch.spark.sql.EsSparkSQL$.saveToEs(EsSparkSQL.scala:97) | |
| at org.elasticsearch.spark.sql.ElasticsearchRelation.insert(DefaultSource.scala:518) | |
| at org.elasticsearch.spark.sql.DefaultSource.createRelation(DefaultSource.scala:93) | |
| at org.apache.spark.sql.execution.datasources.DataSource.write(DataSource.scala:469) | |
| at org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand.run(SaveIntoDataSourceCommand.scala:50) | |
| at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:58) | |
| at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:56) | |
| at org.apache.spark.sql.execution.command.ExecutedCommandExec.doExecute(commands.scala:74) | |
| at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:117) | |
| at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:117) | |
| at org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$1.apply(SparkPlan.scala:138) | |
| at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) | |
| at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:135) | |
| at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:116) | |
| at org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute(QueryExecution.scala:92) | |
| at org.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:92) | |
| at org.apache.spark.sql.DataFrameWriter.runCommand(DataFrameWriter.scala:609) | |
| at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:233) | |
| at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:217) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) | |
| at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) | |
| at java.lang.reflect.Method.invoke(Method.java:498) | |
| at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) | |
| at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) | |
| at py4j.Gateway.invoke(Gateway.java:280) | |
| at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) | |
| at py4j.commands.CallCommand.execute(CallCommand.java:79) | |
| at py4j.GatewayConnection.run(GatewayConnection.java:214) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: java.lang.ClassNotFoundException: org.elasticsearch.spark.sql.EsSparkSQL$$anonfun$saveToEs$1 | |
| at java.net.URLClassLoader.findClass(URLClassLoader.java:381) | |
| at java.lang.ClassLoader.loadClass(ClassLoader.java:424) | |
| at java.lang.ClassLoader.loadClass(ClassLoader.java:357) | |
| at java.lang.Class.forName0(Native Method) | |
| at java.lang.Class.forName(Class.java:348) | |
| at org.apache.spark.serializer.JavaDeserializationStream$$anon$1.resolveClass(JavaSerializer.scala:67) | |
| at java.io.ObjectInputStream.readNonProxyDesc(ObjectInputStream.java:1866) | |
| at java.io.ObjectInputStream.readClassDesc(ObjectInputStream.java:1749) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2040) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1571) | |
| at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:2285) | |
| at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2209) | |
| at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2067) | |
| at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1571) | |
| at java.io.ObjectInputStream.readObject(ObjectInputStream.java:431) | |
| at org.apache.spark.serializer.JavaDeserializationStream.readObject(JavaSerializer.scala:75) | |
| at org.apache.spark.serializer.JavaSerializerInstance.deserialize(JavaSerializer.scala:114) | |
| at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:80) | |
| at org.apache.spark.scheduler.Task.run(Task.scala:108) | |
| at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:338) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| ... 1 more | |
| 18/12/13 05:32:51 INFO org.spark_project.jetty.server.AbstractConnector: Stopped Spark@1bcaa10{HTTP/1.1,[http/1.1]}{0.0.0.0:4041} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment