kryo serialization bug, based off of java-manta-examples/src/main/java/ClientEncryptionServerMultipart.java

01-deserialization_exception.md

If you run App.java with the "all" argument you should get no errors. "all-with-serialization" also works because the pointer-to-native-memory that gets serialized is still valid as long as the JVM doesn't terminate.

If you run App.java once with "initiate" and then subsequently with "complete" you will receive the following exception only if libnss is in use:

Exception in thread "main" com.esotericsoftware.kryo.KryoException: Error during Java deserialization.
Serialization trace:
sessionRef (sun.security.pkcs11.Session)
session (sun.security.pkcs11.SessionKeyRef)
sessionKeyRef (sun.security.pkcs11.P11Key$P11SecretKey)
p11Key (sun.security.pkcs11.P11Cipher)
cipher (com.joyent.manta.client.crypto.EncryptionContext)
encryptionState (com.joyent.manta.client.multipart.EncryptedMultipartUpload)
	at com.esotericsoftware.kryo.serializers.JavaSerializer.read(JavaSerializer.java:65)
	at com.esotericsoftware.kryo.Kryo.readClassAndObject(Kryo.java:813)
	at com.joyent.manta.serialization.SessionRefSerializer.read(SessionRefSerializer.java:80)
	at com.esotericsoftware.kryo.Kryo.readObjectOrNull(Kryo.java:782)
	at com.esotericsoftware.kryo.serializers.ObjectField.read(ObjectField.java:132)
	at com.esotericsoftware.kryo.serializers.FieldSerializer.read(FieldSerializer.java:540)
	at com.esotericsoftware.kryo.Kryo.readObjectOrNull(Kryo.java:782)
	at com.esotericsoftware.kryo.serializers.ObjectField.read(ObjectField.java:132)
	at com.esotericsoftware.kryo.serializers.FieldSerializer.read(FieldSerializer.java:540)
	at com.esotericsoftware.kryo.Kryo.readObjectOrNull(Kryo.java:782)
	at com.esotericsoftware.kryo.serializers.ObjectField.read(ObjectField.java:132)
	at com.esotericsoftware.kryo.serializers.FieldSerializer.read(FieldSerializer.java:540)
	at com.esotericsoftware.kryo.Kryo.readObject(Kryo.java:731)
	at com.esotericsoftware.kryo.serializers.ObjectField.read(ObjectField.java:125)
	at com.esotericsoftware.kryo.serializers.CompatibleFieldSerializer.read(CompatibleFieldSerializer.java:147)
	at com.esotericsoftware.kryo.Kryo.readClassAndObject(Kryo.java:813)
	at com.joyent.manta.serialization.CipherSerializer.read(CipherSerializer.java:225)
	at com.joyent.manta.serialization.CipherSerializer.read(CipherSerializer.java:37)
	at com.esotericsoftware.kryo.Kryo.readObject(Kryo.java:731)
	at com.esotericsoftware.kryo.serializers.ObjectField.read(ObjectField.java:125)
	at com.esotericsoftware.kryo.serializers.FieldSerializer.read(FieldSerializer.java:540)
	at com.esotericsoftware.kryo.Kryo.readClassAndObject(Kryo.java:813)
	at com.joyent.manta.serialization.EncryptionStateSerializer.read(EncryptionStateSerializer.java:158)
	at com.joyent.manta.serialization.EncryptionStateSerializer.read(EncryptionStateSerializer.java:42)
	at com.esotericsoftware.kryo.Kryo.readObject(Kryo.java:731)
	at com.esotericsoftware.kryo.serializers.ObjectField.read(ObjectField.java:125)
	at com.esotericsoftware.kryo.serializers.FieldSerializer.read(FieldSerializer.java:540)
	at com.esotericsoftware.kryo.Kryo.readClassAndObject(Kryo.java:813)
	at com.joyent.manta.serialization.EncryptedMultipartUploaSerializationHelper.deserialize(EncryptedMultipartUploaSerializationHelper.java:263)
	at co.tjcelaya.sandbox.App.multipartUpload(App.java:178)
	at co.tjcelaya.sandbox.App.main(App.java:116)
Caused by: java.io.NotSerializableException: Could not find token
	at sun.security.pkcs11.Token$TokenRep.readResolve(Token.java:446)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at java.io.ObjectStreamClass.invokeReadResolve(ObjectStreamClass.java:1148)
	at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2036)
	at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1535)
	at java.io.ObjectInputStream.readObject(ObjectInputStream.java:422)
	at com.esotericsoftware.kryo.serializers.JavaSerializer.read(JavaSerializer.java:63)
	... 30 more

02-App.java

package co.tjcelaya.sandbox;

import com.esotericsoftware.kryo.Kryo;
import com.joyent.manta.client.MantaClient;
import com.joyent.manta.client.crypto.AesCtrCipherDetails;
import com.joyent.manta.client.crypto.MantaEncryptedObjectInputStream;
import com.joyent.manta.client.crypto.SecretKeyUtils;
import com.joyent.manta.client.crypto.SupportedCipherDetails;
import com.joyent.manta.client.multipart.EncryptedMultipartUpload;
import com.joyent.manta.client.multipart.EncryptedServerSideMultipartManager;
import com.joyent.manta.client.multipart.MantaMultipartUploadPart;
import com.joyent.manta.client.multipart.MantaMultipartUploadTuple;
import com.joyent.manta.client.multipart.ServerSideMultipartUpload;
import com.joyent.manta.config.ChainedConfigContext;
import com.joyent.manta.config.ConfigContext;
import com.joyent.manta.config.DefaultsConfigContext;
import com.joyent.manta.config.EncryptionAuthenticationMode;
import com.joyent.manta.config.EnvVarConfigContext;
import com.joyent.manta.serialization.EncryptedMultipartUploaSerializationHelper;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.io.output.NullOutputStream;
import org.apache.commons.lang3.SerializationUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.exception.ContextedRuntimeException;

import javax.crypto.SecretKey;
import java.io.File;
import java.io.IOException;
import java.util.Arrays;
import java.util.Base64;
import java.util.stream.Stream;

import static java.lang.System.out;

/*
* Usage: set the mantaUsername, privateKeyPath, publicKeyId, and multipartServer with your own values.
 */
public class App {

    private static byte[] keyBytes = Base64.getDecoder().decode("qAnCNUmmFjUTtImNGv241Q==");
    private static SupportedCipherDetails cipherDetails = AesCtrCipherDetails.INSTANCE_128_BIT;
    private static SecretKey secretKey = SecretKeyUtils.loadKey(keyBytes, cipherDetails);
    private static Kryo kryo = new Kryo();
    private static EncryptedMultipartUploaSerializationHelper<ServerSideMultipartUpload> helper =
            new EncryptedMultipartUploaSerializationHelper<>(kryo, secretKey, cipherDetails, ServerSideMultipartUpload.class);

    private static String serializedStatePath = System.getProperty("user.dir") + File.separator + "mpu.state";
    private static String uploadPath = "/tomas.celaya/stor/mpu-serialization-verification";


    public static void main(String... args) {

        ConfigContext config = new ChainedConfigContext(
                new DefaultsConfigContext(),
                new EnvVarConfigContext())
                .setClientEncryptionEnabled(true)
                .setEncryptionAlgorithm("AES256/CTR/NoPadding")
                .setEncryptionAuthenticationMode(EncryptionAuthenticationMode.Optional)
                .setPermitUnencryptedDownloads(false)
                .setEncryptionKeyId("simple/example")
                .setEncryptionPrivateKeyBytes(Base64.getDecoder().decode("RkZGRkZGRkJEOTY3ODNDNkM5MUUyMjIyMTExMTIyMjI="));

        try (MantaClient client = new MantaClient(config)) {
            EncryptedServerSideMultipartManager multipart = new EncryptedServerSideMultipartManager(client);


            if (args[0].equalsIgnoreCase("all")) {
                // initiate
                out.println("initiating");
                cleanUpload(client);
                cleanSerializedState();

                EncryptedMultipartUpload<ServerSideMultipartUpload> upload =
                        multipart.initiateUpload(uploadPath);
                MantaMultipartUploadPart inmemPart1 =
                        multipartUpload(uploadPath, serializedStatePath, multipart, upload, null, true);

                // complete
                out.println("completing");
                multipartUpload(uploadPath, serializedStatePath, multipart, upload, inmemPart1, false);
                cleanSerializedState();

                // validate
                out.println("validating");
                validate(client);
            } else if (args[0].equalsIgnoreCase("all-with-serialization")) {
                // initiate
                out.println("initiating");
                cleanUpload(client);
                cleanSerializedState();

                EncryptedMultipartUpload<ServerSideMultipartUpload> upload =
                        multipart.initiateUpload(uploadPath);
                MantaMultipartUploadPart inmemPart1 =
                        multipartUpload(uploadPath, serializedStatePath, multipart, upload, null, true);

                // complete, don't pass along upload or part1
                out.println("completing");
                multipartUpload(uploadPath, serializedStatePath, multipart, null, null, false);
                cleanSerializedState();

                // validate
                out.println("validating");
                validate(client);
            } else if (args[0].equalsIgnoreCase("initiate")) {
                out.println("initiating");
                cleanUpload(client);
                cleanSerializedState();

                EncryptedMultipartUpload<ServerSideMultipartUpload> upload =
                        multipart.initiateUpload(uploadPath);
                multipartUpload(uploadPath, serializedStatePath, multipart, upload, null, true);
            } else if (args[0].equalsIgnoreCase("complete")) {
                out.println("completing");
                multipartUpload(uploadPath, serializedStatePath, multipart, null, null, false);
                cleanSerializedState();
            } else if (args[0].equalsIgnoreCase("validate")) {
                out.println("validating");
                validate(client);
            }
        } catch (IOException e) {
            e.printStackTrace(System.err);
        }
    }

    private static void validate(MantaClient client) throws IOException {
        try (MantaEncryptedObjectInputStream in =
                     (MantaEncryptedObjectInputStream) client.getAsInputStream(uploadPath)) {
            IOUtils.copy(in, NullOutputStream.NULL_OUTPUT_STREAM);
        }
        out.println("everything's peachy");
    }

    private static void cleanSerializedState() throws IOException {
        final File f1 = new File(serializedStatePath + ".upload");
        if (f1.exists()) FileUtils.forceDelete(f1);
        final File f2 = new File(serializedStatePath + ".part1");
        if (f2.exists()) FileUtils.forceDelete(f2);
    }

    private static void cleanUpload(MantaClient client) throws IOException {
        if (client.existsAndIsAccessible(uploadPath)) {
            client.delete(uploadPath);
        }
    }

    private static MantaMultipartUploadPart multipartUpload(
            String uploadPath,
            String serializedStatePath, EncryptedServerSideMultipartManager multipart,
            EncryptedMultipartUpload<ServerSideMultipartUpload> existingUpload,
            MantaMultipartUploadPart existingPart1,
            boolean starting) {
        try {
            if (starting) {
                MantaMultipartUploadPart part1 = multipart.uploadPart(existingUpload, 1, StringUtils.repeat('a', 5242880));

                FileUtils.writeByteArrayToFile(new File(serializedStatePath + ".upload"), helper.serialize(existingUpload));
                out.println("serialized upload state to " + serializedStatePath + ".upload");
                FileUtils.writeByteArrayToFile(new File(serializedStatePath + ".part1"), SerializationUtils.serialize(part1));
                out.println("serialized part1 to " + serializedStatePath + ".part1");

                // return part1 in case we're doing everything in one JVM
                return part1;
            }

            // doing part2 and completing

            EncryptedMultipartUpload<ServerSideMultipartUpload> upload;
            MantaMultipartUploadPart part1;
            if (existingUpload != null && existingPart1 != null) {
                upload = existingUpload;
                part1 = existingPart1;
                out.println("completing with provided upload and part1");
            } else {
                part1 = SerializationUtils.deserialize(FileUtils.readFileToByteArray(new File(serializedStatePath + ".part1")));
                out.println("deserialized part1 state from " + serializedStatePath + ".part1");
                upload = helper.deserialize(FileUtils.readFileToByteArray(new File(serializedStatePath + ".upload")));
                out.println("deserialized upload state from " + serializedStatePath + ".upload");
            }

            MantaMultipartUploadPart part2 = multipart.uploadPart(upload, 2, StringUtils.repeat('a', 1000000));
            // Complete the process by instructing Manta to assemble the final object from its parts
            MantaMultipartUploadTuple[] parts = new MantaMultipartUploadTuple[]{part1, part2};
            Stream<MantaMultipartUploadTuple> partsStream = Arrays.stream(parts);
            multipart.complete(upload, partsStream);

            out.println(uploadPath + " is now assembled!");

            return null;
        } catch (IOException e) {
            // This catch block is for general network failures
            // For example, ServerSideMultipartUpload.initiateUpload can throw an IOException

            ContextedRuntimeException exception = new ContextedRuntimeException(
                    "A network error occurred when doing a multipart upload to Manta.");
            exception.setContextValue("path", uploadPath);

            throw exception;
        }
    }
}