Created
August 23, 2017 15:10
-
-
Save PerilousApricot/118a6aaa088fe3ed6e07a36e7e5c794d to your computer and use it in GitHub Desktop.
scala backtrace
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
--------------------------------------------------------------------------- | |
Py4JJavaError Traceback (most recent call last) | |
<ipython-input-1-0f51a3d4d6a8> in <module>() | |
18 print droppedColumn.count() | |
19 df.take(1) | |
---> 20 droppedColumn.take(1) | |
/usr/local/opt/apache-spark/libexec/python/pyspark/sql/dataframe.pyc in take(self, num) | |
474 [Row(age=2, name=u'Alice'), Row(age=5, name=u'Bob')] | |
475 """ | |
--> 476 return self.limit(num).collect() | |
477 | |
478 @since(1.3) | |
/usr/local/opt/apache-spark/libexec/python/pyspark/sql/dataframe.pyc in collect(self) | |
436 """ | |
437 with SCCallSiteSync(self._sc) as css: | |
--> 438 port = self._jdf.collectToPython() | |
439 return list(_load_from_socket(port, BatchedSerializer(PickleSerializer()))) | |
440 | |
/usr/local/opt/apache-spark/libexec/python/lib/py4j-0.10.4-src.zip/py4j/java_gateway.py in __call__(self, *args) | |
1131 answer = self.gateway_client.send_command(command) | |
1132 return_value = get_return_value( | |
-> 1133 answer, self.gateway_client, self.target_id, self.name) | |
1134 | |
1135 for temp_arg in temp_args: | |
/usr/local/opt/apache-spark/libexec/python/pyspark/sql/utils.pyc in deco(*a, **kw) | |
61 def deco(*a, **kw): | |
62 try: | |
---> 63 return f(*a, **kw) | |
64 except py4j.protocol.Py4JJavaError as e: | |
65 s = e.java_exception.toString() | |
/usr/local/opt/apache-spark/libexec/python/lib/py4j-0.10.4-src.zip/py4j/protocol.py in get_return_value(answer, gateway_client, target_id, name) | |
317 raise Py4JJavaError( | |
318 "An error occurred while calling {0}{1}{2}.\n". | |
--> 319 format(target_id, ".", name), value) | |
320 else: | |
321 raise Py4JError( | |
Py4JJavaError: An error occurred while calling o37.collectToPython. | |
: org.apache.spark.SparkException: Job aborted due to stage failure: Task 2 in stage 11.0 failed 1 times, most recent failure: Lost task 2.0 in stage 11.0 (TID 42, localhost, executor driver): scala.MatchError: 0 (of class java.lang.Integer) | |
at org.apache.spark.sql.catalyst.CatalystTypeConverters$StringConverter$.toCatalystImpl(CatalystTypeConverters.scala:276) | |
at org.apache.spark.sql.catalyst.CatalystTypeConverters$StringConverter$.toCatalystImpl(CatalystTypeConverters.scala:275) | |
at org.apache.spark.sql.catalyst.CatalystTypeConverters$CatalystTypeConverter.toCatalyst(CatalystTypeConverters.scala:103) | |
at org.apache.spark.sql.catalyst.CatalystTypeConverters$ArrayConverter$$anonfun$toCatalystImpl$2.apply(CatalystTypeConverters.scala:164) | |
at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) | |
at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) | |
at scala.collection.Iterator$class.foreach(Iterator.scala:893) | |
at scala.collection.AbstractIterator.foreach(Iterator.scala:1336) | |
at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) | |
at scala.collection.AbstractIterable.foreach(Iterable.scala:54) | |
at scala.collection.TraversableLike$class.map(TraversableLike.scala:234) | |
at scala.collection.AbstractTraversable.map(Traversable.scala:104) | |
at org.apache.spark.sql.catalyst.CatalystTypeConverters$ArrayConverter.toCatalystImpl(CatalystTypeConverters.scala:164) | |
at org.apache.spark.sql.catalyst.CatalystTypeConverters$ArrayConverter.toCatalystImpl(CatalystTypeConverters.scala:154) | |
at org.apache.spark.sql.catalyst.CatalystTypeConverters$CatalystTypeConverter.toCatalyst(CatalystTypeConverters.scala:103) | |
at org.apache.spark.sql.catalyst.CatalystTypeConverters$$anonfun$createToCatalystConverter$2.apply(CatalystTypeConverters.scala:379) | |
at org.apache.spark.sql.execution.RDDConversions$$anonfun$rowToRowRdd$1$$anonfun$apply$3.apply(ExistingRDD.scala:61) | |
at org.apache.spark.sql.execution.RDDConversions$$anonfun$rowToRowRdd$1$$anonfun$apply$3.apply(ExistingRDD.scala:58) | |
at scala.collection.Iterator$$anon$11.next(Iterator.scala:409) | |
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIterator.processNext(Unknown Source) | |
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43) | |
at org.apache.spark.sql.execution.WholeStageCodegenExec$$anonfun$8$$anon$1.hasNext(WholeStageCodegenExec.scala:395) | |
at org.apache.spark.sql.execution.SparkPlan$$anonfun$2.apply(SparkPlan.scala:234) | |
at org.apache.spark.sql.execution.SparkPlan$$anonfun$2.apply(SparkPlan.scala:228) | |
at org.apache.spark.rdd.RDD$$anonfun$mapPartitionsInternal$1$$anonfun$apply$25.apply(RDD.scala:827) | |
at org.apache.spark.rdd.RDD$$anonfun$mapPartitionsInternal$1$$anonfun$apply$25.apply(RDD.scala:827) | |
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38) | |
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:323) | |
at org.apache.spark.rdd.RDD.iterator(RDD.scala:287) | |
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87) | |
at org.apache.spark.scheduler.Task.run(Task.scala:108) | |
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:335) | |
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
at java.lang.Thread.run(Thread.java:748) | |
Driver stacktrace: | |
at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1499) | |
at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1487) | |
at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1486) | |
at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) | |
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) | |
at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1486) | |
at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:814) | |
at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:814) | |
at scala.Option.foreach(Option.scala:257) | |
at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:814) | |
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:1714) | |
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1669) | |
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1658) | |
at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48) | |
at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:630) | |
at org.apache.spark.SparkContext.runJob(SparkContext.scala:2022) | |
at org.apache.spark.SparkContext.runJob(SparkContext.scala:2043) | |
at org.apache.spark.SparkContext.runJob(SparkContext.scala:2062) | |
at org.apache.spark.sql.execution.SparkPlan.executeTake(SparkPlan.scala:336) | |
at org.apache.spark.sql.execution.CollectLimitExec.executeCollect(limit.scala:38) | |
at org.apache.spark.sql.Dataset$$anonfun$collectToPython$1.apply$mcI$sp(Dataset.scala:2803) | |
at org.apache.spark.sql.Dataset$$anonfun$collectToPython$1.apply(Dataset.scala:2800) | |
at org.apache.spark.sql.Dataset$$anonfun$collectToPython$1.apply(Dataset.scala:2800) | |
at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:65) | |
at org.apache.spark.sql.Dataset.withNewExecutionId(Dataset.scala:2823) | |
at org.apache.spark.sql.Dataset.collectToPython(Dataset.scala:2800) | |
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) | |
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) | |
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) | |
at java.lang.reflect.Method.invoke(Method.java:498) | |
at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) | |
at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) | |
at py4j.Gateway.invoke(Gateway.java:280) | |
at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) | |
at py4j.commands.CallCommand.execute(CallCommand.java:79) | |
at py4j.GatewayConnection.run(GatewayConnection.java:214) | |
at java.lang.Thread.run(Thread.java:748) | |
Caused by: scala.MatchError: 0 (of class java.lang.Integer) | |
at org.apache.spark.sql.catalyst.CatalystTypeConverters$StringConverter$.toCatalystImpl(CatalystTypeConverters.scala:276) | |
at org.apache.spark.sql.catalyst.CatalystTypeConverters$StringConverter$.toCatalystImpl(CatalystTypeConverters.scala:275) | |
at org.apache.spark.sql.catalyst.CatalystTypeConverters$CatalystTypeConverter.toCatalyst(CatalystTypeConverters.scala:103) | |
at org.apache.spark.sql.catalyst.CatalystTypeConverters$ArrayConverter$$anonfun$toCatalystImpl$2.apply(CatalystTypeConverters.scala:164) | |
at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) | |
at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) | |
at scala.collection.Iterator$class.foreach(Iterator.scala:893) | |
at scala.collection.AbstractIterator.foreach(Iterator.scala:1336) | |
at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) | |
at scala.collection.AbstractIterable.foreach(Iterable.scala:54) | |
at scala.collection.TraversableLike$class.map(TraversableLike.scala:234) | |
at scala.collection.AbstractTraversable.map(Traversable.scala:104) | |
at org.apache.spark.sql.catalyst.CatalystTypeConverters$ArrayConverter.toCatalystImpl(CatalystTypeConverters.scala:164) | |
at org.apache.spark.sql.catalyst.CatalystTypeConverters$ArrayConverter.toCatalystImpl(CatalystTypeConverters.scala:154) | |
at org.apache.spark.sql.catalyst.CatalystTypeConverters$CatalystTypeConverter.toCatalyst(CatalystTypeConverters.scala:103) | |
at org.apache.spark.sql.catalyst.CatalystTypeConverters$$anonfun$createToCatalystConverter$2.apply(CatalystTypeConverters.scala:379) | |
at org.apache.spark.sql.execution.RDDConversions$$anonfun$rowToRowRdd$1$$anonfun$apply$3.apply(ExistingRDD.scala:61) | |
at org.apache.spark.sql.execution.RDDConversions$$anonfun$rowToRowRdd$1$$anonfun$apply$3.apply(ExistingRDD.scala:58) | |
at scala.collection.Iterator$$anon$11.next(Iterator.scala:409) | |
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIterator.processNext(Unknown Source) | |
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43) | |
at org.apache.spark.sql.execution.WholeStageCodegenExec$$anonfun$8$$anon$1.hasNext(WholeStageCodegenExec.scala:395) | |
at org.apache.spark.sql.execution.SparkPlan$$anonfun$2.apply(SparkPlan.scala:234) | |
at org.apache.spark.sql.execution.SparkPlan$$anonfun$2.apply(SparkPlan.scala:228) | |
at org.apache.spark.rdd.RDD$$anonfun$mapPartitionsInternal$1$$anonfun$apply$25.apply(RDD.scala:827) | |
at org.apache.spark.rdd.RDD$$anonfun$mapPartitionsInternal$1$$anonfun$apply$25.apply(RDD.scala:827) | |
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38) | |
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:323) | |
at org.apache.spark.rdd.RDD.iterator(RDD.scala:287) | |
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87) | |
at org.apache.spark.scheduler.Task.run(Task.scala:108) | |
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:335) | |
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
... 1 more |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment