Created
November 29, 2017 11:03
-
-
Save avaranovich/b9f1a10ddbffc41ee49cba40ad618b15 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --------------------------------------------------------------------------- | |
| Py4JJavaError Traceback (most recent call last) | |
| <command-634889928058462> in <module>() | |
| 17 #display(df.limit(5)) | |
| 18 | |
| ---> 19 toCSV = filter(None, list(map(xml_to_dict, xml.collect()))) | |
| 20 keys = toCSV[0].keys() | |
| 21 #print(keys) | |
| /databricks/spark/python/pyspark/rdd.py in collect(self) | |
| 807 """ | |
| 808 with SCCallSiteSync(self.context) as css: | |
| --> 809 port = self.ctx._jvm.PythonRDD.collectAndServe(self._jrdd.rdd()) | |
| 810 return list(_load_from_socket(port, self._jrdd_deserializer)) | |
| 811 | |
| /databricks/spark/python/lib/py4j-0.10.4-src.zip/py4j/java_gateway.py in __call__(self, *args) | |
| 1131 answer = self.gateway_client.send_command(command) | |
| 1132 return_value = get_return_value( | |
| -> 1133 answer, self.gateway_client, self.target_id, self.name) | |
| 1134 | |
| 1135 for temp_arg in temp_args: | |
| /databricks/spark/python/pyspark/sql/utils.py in deco(*a, **kw) | |
| 61 def deco(*a, **kw): | |
| 62 try: | |
| ---> 63 return f(*a, **kw) | |
| 64 except py4j.protocol.Py4JJavaError as e: | |
| 65 s = e.java_exception.toString() | |
| /databricks/spark/python/lib/py4j-0.10.4-src.zip/py4j/protocol.py in get_return_value(answer, gateway_client, target_id, name) | |
| 317 raise Py4JJavaError( | |
| 318 "An error occurred while calling {0}{1}{2}.\n". | |
| --> 319 format(target_id, ".", name), value) | |
| 320 else: | |
| 321 raise Py4JError( | |
| Py4JJavaError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.collectAndServe. | |
| : org.apache.spark.SparkException: Job aborted due to stage failure: Task 1615 in stage 56.0 failed 4 times, most recent failure: Lost task 1615.3 in stage 56.0 (TID 8786, 10.139.64.7, executor 5): java.lang.IllegalArgumentException: No value for dfs.adls.oauth2.access.token.provider found in conf file. | |
| at com.databricks.adl.AdlFileSystem.getNonEmptyVal(AdlFileSystem.java:734) | |
| at com.databricks.adl.AdlFileSystem.getCustomAccessTokenProvider(AdlFileSystem.java:196) | |
| at com.databricks.adl.AdlFileSystem.getAccessTokenProvider(AdlFileSystem.java:230) | |
| at com.databricks.adl.AdlFileSystem.initialize(AdlFileSystem.java:140) | |
| at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2669) | |
| at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:94) | |
| at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2703) | |
| at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2685) | |
| at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:373) | |
| at org.apache.hadoop.fs.Path.getFileSystem(Path.java:295) | |
| at org.apache.hadoop.mapred.LineRecordReader.<init>(LineRecordReader.java:108) | |
| at org.apache.hadoop.mapred.TextInputFormat.getRecordReader(TextInputFormat.java:67) | |
| at org.apache.spark.rdd.HadoopRDD$$anon$1.liftedTree1$1(HadoopRDD.scala:251) | |
| at org.apache.spark.rdd.HadoopRDD$$anon$1.<init>(HadoopRDD.scala:250) | |
| at org.apache.spark.rdd.HadoopRDD.compute(HadoopRDD.scala:208) | |
| at org.apache.spark.rdd.HadoopRDD.compute(HadoopRDD.scala:94) | |
| at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:324) | |
| at org.apache.spark.rdd.RDD.iterator(RDD.scala:288) | |
| at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38) | |
| at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:324) | |
| at org.apache.spark.rdd.RDD.iterator(RDD.scala:288) | |
| at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87) | |
| at org.apache.spark.scheduler.Task.run(Task.scala:108) | |
| at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:349) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Driver stacktrace: | |
| at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1677) | |
| at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1665) | |
| at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1664) | |
| at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) | |
| at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) | |
| at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1664) | |
| at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:931) | |
| at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:931) | |
| at scala.Option.foreach(Option.scala:257) | |
| at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:931) | |
| at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:1897) | |
| at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1848) | |
| at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1836) | |
| at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48) | |
| at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:733) | |
| at org.apache.spark.SparkContext.runJob(SparkContext.scala:2060) | |
| at org.apache.spark.SparkContext.runJob(SparkContext.scala:2081) | |
| at org.apache.spark.SparkContext.runJob(SparkContext.scala:2100) | |
| at org.apache.spark.SparkContext.runJob(SparkContext.scala:2125) | |
| at org.apache.spark.rdd.RDD$$anonfun$collect$1.apply(RDD.scala:937) | |
| at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) | |
| at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112) | |
| at org.apache.spark.rdd.RDD.withScope(RDD.scala:363) | |
| at org.apache.spark.rdd.RDD.collect(RDD.scala:936) | |
| at org.apache.spark.api.python.PythonRDD$.collectAndServe(PythonRDD.scala:178) | |
| at org.apache.spark.api.python.PythonRDD.collectAndServe(PythonRDD.scala) | |
| at sun.reflect.GeneratedMethodAccessor324.invoke(Unknown Source) | |
| at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) | |
| at java.lang.reflect.Method.invoke(Method.java:498) | |
| at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) | |
| at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380) | |
| at py4j.Gateway.invoke(Gateway.java:293) | |
| at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) | |
| at py4j.commands.CallCommand.execute(CallCommand.java:79) | |
| at py4j.GatewayConnection.run(GatewayConnection.java:226) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: java.lang.IllegalArgumentException: No value for dfs.adls.oauth2.access.token.provider found in conf file. | |
| at com.databricks.adl.AdlFileSystem.getNonEmptyVal(AdlFileSystem.java:734) | |
| at com.databricks.adl.AdlFileSystem.getCustomAccessTokenProvider(AdlFileSystem.java:196) | |
| at com.databricks.adl.AdlFileSystem.getAccessTokenProvider(AdlFileSystem.java:230) | |
| at com.databricks.adl.AdlFileSystem.initialize(AdlFileSystem.java:140) | |
| at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2669) | |
| at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:94) | |
| at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2703) | |
| at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2685) | |
| at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:373) | |
| at org.apache.hadoop.fs.Path.getFileSystem(Path.java:295) | |
| at org.apache.hadoop.mapred.LineRecordReader.<init>(LineRecordReader.java:108) | |
| at org.apache.hadoop.mapred.TextInputFormat.getRecordReader(TextInputFormat.java:67) | |
| at org.apache.spark.rdd.HadoopRDD$$anon$1.liftedTree1$1(HadoopRDD.scala:251) | |
| at org.apache.spark.rdd.HadoopRDD$$anon$1.<init>(HadoopRDD.scala:250) | |
| at org.apache.spark.rdd.HadoopRDD.compute(HadoopRDD.scala:208) | |
| at org.apache.spark.rdd.HadoopRDD.compute(HadoopRDD.scala:94) | |
| at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:324) | |
| at org.apache.spark.rdd.RDD.iterator(RDD.scala:288) | |
| at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38) | |
| at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:324) | |
| at org.apache.spark.rdd.RDD.iterator(RDD.scala:288) | |
| at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87) | |
| at org.apache.spark.scheduler.Task.run(Task.scala:108) | |
| at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:349) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| ... 1 more |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment