Decrypt by hand TLS payload

netty_tls_raw.java

  public class Test {
    
    @Test
    public void testExtractMasterkeyWorksCorrectly() throws Exception {

        SelfSignedCertificate cert = new SelfSignedCertificate();
        SslContext serverContext = SslContextBuilder.forServer(cert.key(), cert.cert())
                .sslProvider(SslProvider.OPENSSL).build();
        final OpenSslEngine serverEngine =
                (OpenSslEngine) serverContext.newEngine(UnpooledByteBufAllocator.DEFAULT);

        SslContext clientContext = SslContextBuilder.forClient()
                .trustManager(cert.certificate())
                .sslProvider(SslProvider.OPENSSL).build();
        final OpenSslEngine clientEngine =
                (OpenSslEngine) clientContext.newEngine(UnpooledByteBufAllocator.DEFAULT);

        //lets set the cipher suite to a specific one with DHE

        //https://www.ietf.org/rfc/rfc5289.txt
        //For cipher suites ending with _SHA256, the PRF is the TLS PRF
        //[RFC5246] with SHA-256 as the hash function.  The MAC is HMAC
        //[RFC2104] with SHA-256 as the hash function.
        clientEngine.setEnabledCipherSuites(new String[]{ "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256"});
        serverEngine.setEnabledCipherSuites(new String[]{ "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256"});

        int appBufferMax = clientEngine.getSession().getApplicationBufferSize();
        int netBufferMax = clientEngine.getSession().getPacketBufferSize();

        /*
         * We'll make the input buffers a bit bigger than the max needed
         * size, so that unwrap()s following a successful data transfer
         * won't generate BUFFER_OVERFLOWS.
         */
        ByteBuffer clientIn = ByteBuffer.allocate(appBufferMax + 50);
        ByteBuffer serverIn = ByteBuffer.allocate(appBufferMax + 50);

        ByteBuffer cTOs = ByteBuffer.allocate(netBufferMax);
        ByteBuffer sTOc = ByteBuffer.allocate(netBufferMax);

        ByteBuffer clientOut = ByteBuffer.wrap("Hi Server, I'm Client".getBytes());
        ByteBuffer serverOut = ByteBuffer.wrap("Hello Client, I'm Server".getBytes());

        // This implementation is largely imitated from
        // https://docs.oracle.com/javase/8/docs/technotes/guides/security/jsse/samples/sslengine/SSLEngineSimpleDemo.java
        // It has been simplified however without the need for running delegation tasks, as of now,
        // OpenSslEngine does not do unwrap/wrap non-blocking

        // Do handshake for SSL
        // A typical handshake will usually contain the following steps:
        // 1. wrap:     ClientHello
        // 2. unwrap:   ServerHello/Cert/ServerHelloDone
        // 3. wrap:     ClientKeyExchange
        // 4. wrap:     ChangeCipherSpec
        // 5. wrap:     Finished
        // 6. unwrap:   ChangeCipherSpec
        // 7. unwrap:   Finished

        //set a for loop; instead of a while loop to guarantee we quit out eventually
        boolean asserted = false;
        for (int i = 0; i < 1000; i++) {

            clientEngine.wrap(clientOut, cTOs);
            serverEngine.wrap(serverOut, sTOc);

            cTOs.flip();
            sTOc.flip();

            clientEngine.unwrap(sTOc, clientIn);
            serverEngine.unwrap(cTOs, serverIn);

            // check when the application data has fully been consumed and sent
            // for both the client and server
            if ((clientOut.limit() == serverIn.position()) &&
                    (serverOut.limit() == clientIn.position())) {
                byte[] serverRandom = SSL.getServerRandom(serverEngine.sslPointer());
                byte[] clientRandom = SSL.getClientRandom(clientEngine.sslPointer());
                byte[] serverMasterKey = SSL.getMasterKey(serverEngine.sslPointer());
                byte[] clientMasterKey = SSL.getMasterKey(clientEngine.sslPointer());
                asserted = true;
                assertArrayEquals(serverMasterKey, clientMasterKey);

                // let us re-read the encrypted data and decrypt it ourselves!
                cTOs.flip();
                sTOc.flip();


                // See http://tools.ietf.org/html/rfc5246#section-6.3:
                // key_block = PRF(SecurityParameters.master_secret, "key expansion",
                //                 SecurityParameters.server_random + SecurityParameters.client_random);
                //
                // partitioned:
                //       client_write_MAC_secret[SecurityParameters.hash_size]
                //       server_write_MAC_secret[SecurityParameters.hash_size]
                //       client_write_key[SecurityParameters.key_material_length]
                //       server_write_key[SecurityParameters.key_material_length]

                int keySize = 16; // AES is 16 bytes or 128 bits
                int macSize = 32; // SHA256 is 32 bytes or 256 bits
                int keyBlockSize = (2 * keySize) + (2 * macSize);

                byte[] keyBlock = doPRF(serverMasterKey, "key expansion".getBytes(),
                        ArrayUtils.addAll(serverRandom, clientRandom), keyBlockSize);
                int offset = 0;
                byte[] clientWriteMac = Arrays.copyOfRange(keyBlock, offset, offset + macSize);
                offset += macSize;

                byte[] serverWriteMac = Arrays.copyOfRange(keyBlock, offset, offset + macSize);
                offset += macSize;

                byte[] clientWriteKey = Arrays.copyOfRange(keyBlock, offset, offset + keySize);
                offset += keySize;

                byte[] serverWriteKey = Arrays.copyOfRange(keyBlock, offset, offset + keySize);
                offset += keySize;

                //advance the cipher text by 5
                //to take into account the TLS Record Heade
                cTOs.get(new byte[5]);

                byte[] ciphertext = new byte[cTOs.remaining()];
                cTOs.get(ciphertext);

                //the initialization vector is the first 16 bytes (128 bits) if the payload
                byte[] clientWriteIV = Arrays.copyOfRange(ciphertext, 0, 16);
                ciphertext = Arrays.copyOfRange(ciphertext, 16, ciphertext.length);

                SecretKeySpec secretKey = new SecretKeySpec(clientWriteKey, "AES");
                final IvParameterSpec ivForCBC = new IvParameterSpec(clientWriteIV);
                Cipher cipher = Cipher.getInstance("AES/CBC/NoPadding");
                cipher.init(Cipher.DECRYPT_MODE, secretKey, ivForCBC);
                byte[] plaintext = cipher.doFinal(ciphertext);
                assertArrayEquals(plaintext, "Hi Server, I'm Client".getBytes());

                break;
            } else {
                cTOs.compact();
                sTOc.compact();
            }
        }

        assertTrue("The assertions were never executed.", asserted);
    }

    private static byte[] doPRF(byte[] secret, byte[] label, byte[] seed, int length) throws Exception {
        Mac hmac = Mac.getInstance("HmacSHA256");
        hmac.init(new SecretKeySpec(secret, "HmacSHA256"));
        /*
         * P_hash(secret, seed) = HMAC_hash(secret, A(1) + seed) +
         * HMAC_hash(secret, A(2) + seed) + HMAC_hash(secret, A(3) + seed) + ...
         * where + indicates concatenation. A() is defined as: A(0) = seed, A(i)
         * = HMAC_hash(secret, A(i-1))
         */

        int iterations = (int) Math.ceil(length / (double) hmac.getMacLength());
        byte[] expansion = new byte[0];
        byte[] data = ArrayUtils.addAll(label, seed);
        byte[] A = data;
        for (int i = 0; i < iterations; i++) {
            A = hmac.doFinal(A);
            expansion = ArrayUtils.addAll(expansion, hmac.doFinal(ArrayUtils.addAll(A, data)));
        }
        return Arrays.copyOf(expansion, length);
    }
  }