Created
October 10, 2016 18:27
-
-
Save joey/d3bf040af31e854b3be374e2c016d7e1 to your computer and use it in GitHub Desktop.
Serialization error with mapWithState
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Serialization trace: | |
| stateMap (org.apache.spark.streaming.rdd.MapWithStateRDDRecord) | |
| at com.esotericsoftware.kryo.serializers.JavaSerializer.write(JavaSerializer.java:34) | |
| at com.esotericsoftware.kryo.Kryo.writeObject(Kryo.java:501) | |
| at com.esotericsoftware.kryo.serializers.FieldSerializer$ObjectField.write(FieldSerializer.java:564) | |
| at com.esotericsoftware.kryo.serializers.FieldSerializer.write(FieldSerializer.java:213) | |
| at com.esotericsoftware.kryo.Kryo.writeClassAndObject(Kryo.java:568) | |
| at org.apache.spark.serializer.KryoSerializationStream.writeObject(KryoSerializer.scala:194) | |
| at org.apache.spark.serializer.SerializationStream.writeAll(Serializer.scala:153) | |
| at org.apache.spark.rdd.ReliableCheckpointRDD$$anonfun$writePartitionToCheckpointFile$2.apply(ReliableCheckpointRDD.scala:182) | |
| at org.apache.spark.rdd.ReliableCheckpointRDD$$anonfun$writePartitionToCheckpointFile$2.apply(ReliableCheckpointRDD.scala:182) | |
| at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1250) | |
| at org.apache.spark.rdd.ReliableCheckpointRDD$.writePartitionToCheckpointFile(ReliableCheckpointRDD.scala:183) | |
| at org.apache.spark.rdd.ReliableCheckpointRDD$$anonfun$writeRDDToCheckpointDirectory$1.apply(ReliableCheckpointRDD.scala:136) | |
| at org.apache.spark.rdd.ReliableCheckpointRDD$$anonfun$writeRDDToCheckpointDirectory$1.apply(ReliableCheckpointRDD.scala:136) | |
| at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) | |
| at org.apache.spark.scheduler.Task.run(Task.scala:89) | |
| at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:213) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) | |
| at java.lang.Thread.run(Thread.java:745) | |
| Caused by: java.io.NotSerializableException: com.rocana.data.Tuple | |
| at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1183) | |
| at java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:347) | |
| at org.apache.spark.streaming.util.OpenHashMapBasedStateMap.writeObject(StateMap.scala:246) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) | |
| at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) | |
| at java.lang.reflect.Method.invoke(Method.java:606) | |
| at java.io.ObjectStreamClass.invokeWriteObject(ObjectStreamClass.java:988) | |
| at java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1495) | |
| at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1431) | |
| at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1177) | |
| at java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:347) | |
| at com.esotericsoftware.kryo.serializers.JavaSerializer.write(JavaSerializer.java:31) | |
| ... 18 more |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment