-
-
Save goFrendiAsgard/ea16656e85e5bc9cb256339415ee1a1f to your computer and use it in GitHub Desktop.
apache-spark error
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| [guldan@draenor ~]$ spark-shell | |
| /usr/bin/hadoop | |
| WARNING: HADOOP_SLAVES has been replaced by HADOOP_WORKERS. Using value of HADOOP_SLAVES. | |
| 2019-03-27 18:34:29 WARN NativeCodeLoader:60 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable | |
| Setting default log level to "WARN". | |
| To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel). | |
| WARNING: An illegal reflective access operation has occurred | |
| WARNING: Illegal reflective access by io.netty.util.internal.ReflectionUtil (file:/opt/apache-spark/jars/netty-all-4.1.17.Final.jar) to constructor java.nio.DirectByteBuffer(long,int) | |
| WARNING: Please consider reporting this to the maintainers of io.netty.util.internal.ReflectionUtil | |
| WARNING: Use --illegal-access=warn to enable warnings of further illegal reflective access operations | |
| WARNING: All illegal access operations will be denied in a future release | |
| Spark context Web UI available at http://localhost:4040 | |
| Spark context available as 'sc' (master = local[*], app id = local-1553686480196). | |
| Spark session available as 'spark'. | |
| Welcome to | |
| ____ __ | |
| / __/__ ___ _____/ /__ | |
| _\ \/ _ \/ _ `/ __/ '_/ | |
| /___/ .__/\_,_/_/ /_/\_\ version 2.4.0 | |
| /_/ | |
| Using Scala version 2.11.12 (OpenJDK 64-Bit Server VM, Java 11.0.3) | |
| Type in expressions to have them evaluated. | |
| Type :help for more information. | |
| scala> sc.version | |
| res0: String = 2.4.0 | |
| scala> var Data = sc.textFile("installer.sh") | |
| Data: org.apache.spark.rdd.RDD[String] = installer.sh MapPartitionsRDD[1] at textFile at <console>:24 | |
| scala> var tokens = Data.flatMap(s => s.split(" ")) | |
| tokens: org.apache.spark.rdd.RDD[String] = MapPartitionsRDD[2] at flatMap at <console>:25 | |
| scala> var tokens_1 = tokens.map(s => (s,1)) | |
| tokens_1: org.apache.spark.rdd.RDD[(String, Int)] = MapPartitionsRDD[3] at map at <console>:25 | |
| scala> var sum_each = tokens_1.reduceByKey((a, b) => a + b) | |
| sum_each: org.apache.spark.rdd.RDD[(String, Int)] = ShuffledRDD[4] at reduceByKey at <console>:25 | |
| scala> sum_each.collect() | |
| java.lang.IllegalArgumentException: Unsupported class file major version 55 | |
| at org.apache.xbean.asm6.ClassReader.<init>(ClassReader.java:166) | |
| at org.apache.xbean.asm6.ClassReader.<init>(ClassReader.java:148) | |
| at org.apache.xbean.asm6.ClassReader.<init>(ClassReader.java:136) | |
| at org.apache.xbean.asm6.ClassReader.<init>(ClassReader.java:237) | |
| at org.apache.spark.util.ClosureCleaner$.getClassReader(ClosureCleaner.scala:49) | |
| at org.apache.spark.util.FieldAccessFinder$$anon$3$$anonfun$visitMethodInsn$2.apply(ClosureCleaner.scala:517) | |
| at org.apache.spark.util.FieldAccessFinder$$anon$3$$anonfun$visitMethodInsn$2.apply(ClosureCleaner.scala:500) | |
| at scala.collection.TraversableLike$WithFilter$$anonfun$foreach$1.apply(TraversableLike.scala:733) | |
| at scala.collection.mutable.HashMap$$anon$1$$anonfun$foreach$2.apply(HashMap.scala:134) | |
| at scala.collection.mutable.HashMap$$anon$1$$anonfun$foreach$2.apply(HashMap.scala:134) | |
| at scala.collection.mutable.HashTable$class.foreachEntry(HashTable.scala:236) | |
| at scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:40) | |
| at scala.collection.mutable.HashMap$$anon$1.foreach(HashMap.scala:134) | |
| at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) | |
| at org.apache.spark.util.FieldAccessFinder$$anon$3.visitMethodInsn(ClosureCleaner.scala:500) | |
| at org.apache.xbean.asm6.ClassReader.readCode(ClassReader.java:2175) | |
| at org.apache.xbean.asm6.ClassReader.readMethod(ClassReader.java:1238) | |
| at org.apache.xbean.asm6.ClassReader.accept(ClassReader.java:631) | |
| at org.apache.xbean.asm6.ClassReader.accept(ClassReader.java:355) | |
| at org.apache.spark.util.ClosureCleaner$$anonfun$org$apache$spark$util$ClosureCleaner$$clean$14.apply(ClosureCleaner.scala:307) | |
| at org.apache.spark.util.ClosureCleaner$$anonfun$org$apache$spark$util$ClosureCleaner$$clean$14.apply(ClosureCleaner.scala:306) | |
| at scala.collection.immutable.List.foreach(List.scala:392) | |
| at org.apache.spark.util.ClosureCleaner$.org$apache$spark$util$ClosureCleaner$$clean(ClosureCleaner.scala:306) | |
| at org.apache.spark.util.ClosureCleaner$.clean(ClosureCleaner.scala:162) | |
| at org.apache.spark.SparkContext.clean(SparkContext.scala:2326) | |
| at org.apache.spark.SparkContext.runJob(SparkContext.scala:2100) | |
| at org.apache.spark.SparkContext.runJob(SparkContext.scala:2126) | |
| at org.apache.spark.rdd.RDD$$anonfun$collect$1.apply(RDD.scala:945) | |
| at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) | |
| at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112) | |
| at org.apache.spark.rdd.RDD.withScope(RDD.scala:363) | |
| at org.apache.spark.rdd.RDD.collect(RDD.scala:944) | |
| ... 49 elided | |
| scala> |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment