RTMP digest handshaking in FFmpeg, annotated source code

rtmpproto.c

/// Annotations start with a `///`, just like this line.

/// In `rtmp_open` function:
///
///     rt->state = STATE_START;
///     if (!rt->listen && (ret = rtmp_handshake(s, rt)) < 0)
///         goto fail;
///     if (rt->listen && (ret = rtmp_server_handshake(s, rt)) < 0)
///         goto fail;
///
/// So `rtmp_handshake` is for FFmpeg's RTMP client to handshake with server.

/**
 * Perform handshake with the server by means of exchanging pseudorandom data
 * signed with HMAC-SHA2 digest.
 *
 * @return 0 if handshake succeeds, negative value otherwise
 */
static int rtmp_handshake(URLContext *s, RTMPContext *rt)
{
    AVLFG rnd;
    uint8_t tosend    [RTMP_HANDSHAKE_PACKET_SIZE+1] = {
        3,                // unencrypted data
        0, 0, 0, 0,       // client uptime
        RTMP_CLIENT_VER1,
        RTMP_CLIENT_VER2,
        RTMP_CLIENT_VER3,
        RTMP_CLIENT_VER4,
    };
    uint8_t clientdata[RTMP_HANDSHAKE_PACKET_SIZE];
    uint8_t serverdata[RTMP_HANDSHAKE_PACKET_SIZE+1];
    int i;
    int server_pos, client_pos;
    uint8_t digest[32], signature[32];
    int ret, type = 0;

    av_log(s, AV_LOG_DEBUG, "Handshaking...\n");

    av_lfg_init(&rnd, 0xDEADC0DE);
    /// Note: it's not 1536 bytes of pseudorandom data, it's (1536 - 8) bytes
    /// Also note that `tosend` is reused when sending C2.
    // generate handshake packet - 1536 bytes of pseudorandom data
    for (i = 9; i <= RTMP_HANDSHAKE_PACKET_SIZE; i++)
        tosend[i] = av_lfg_get(&rnd) >> 24;

    if (CONFIG_FFRTMPCRYPT_PROTOCOL && rt->encrypted) {
        /* When the client wants to use RTMPE, we have to change the command
         * byte to 0x06 which means to use encrypted data and we have to set
         * the flash version to at least 9.0.115.0. */
        tosend[0] = 6;
        tosend[5] = 128;
        tosend[6] = 0;
        tosend[7] = 3;
        tosend[8] = 2;

        /* Initialize the Diffie-Hellmann context and generate the public key
         * to send to the server. */
        if ((ret = ff_rtmpe_gen_pub_key(rt->stream, tosend + 1)) < 0)
            return ret;
    }

    /// `client_pos`: the position of digest field for C1
    /// Here the offset value is random
    client_pos = rtmp_handshake_imprint_with_digest(tosend + 1, rt->encrypted);
    if (client_pos < 0)
        return client_pos;

    /// Send C0+C1 ((1536 + 1)B)
    if ((ret = ffurl_write(rt->stream, tosend,
                           RTMP_HANDSHAKE_PACKET_SIZE + 1)) < 0) {
        av_log(s, AV_LOG_ERROR, "Cannot write RTMP handshake request\n");
        return ret;
    }

    /// Read S0+S1 ((1536 + 1)B)
    if ((ret = ffurl_read_complete(rt->stream, serverdata,
                                   RTMP_HANDSHAKE_PACKET_SIZE + 1)) < 0) {
        av_log(s, AV_LOG_ERROR, "Cannot read RTMP handshake response\n");
        return ret;
    }

    /// According to RTMP spec, The server MUST wait until C1 has been received before
    /// sending S2.

    /// Read S2 (1536 B)
    if ((ret = ffurl_read_complete(rt->stream, clientdata,
                                   RTMP_HANDSHAKE_PACKET_SIZE)) < 0) {
        av_log(s, AV_LOG_ERROR, "Cannot read RTMP handshake response\n");
        return ret;
    }

    /// S0
    av_log(s, AV_LOG_DEBUG, "Type answer %d\n", serverdata[0]);
    /// S1[5..5+4], version
    av_log(s, AV_LOG_DEBUG, "Server version %d.%d.%d.%d\n",
           serverdata[5], serverdata[6], serverdata[7], serverdata[8]);

    /// Prepare C2

    /// `rt->is_input`:
    /// - TRUE: subscriber
    /// - FALSE: publisher

    /// Commonly we have `serverdata[5] >= 3` evaluate to `true`
    if (rt->is_input && serverdata[5] >= 3) {
        /// `server_pos`: the position of digest field for S1
        ///
        /// For key-digest
        server_pos = rtmp_validate_digest(serverdata + 1, 772);
        if (server_pos < 0)
            return server_pos;

        if (!server_pos) {
            type = 1;
            /// For digest-key
            server_pos = rtmp_validate_digest(serverdata + 1, 8);
            if (server_pos < 0)
                return server_pos;

            if (!server_pos) {
                av_log(s, AV_LOG_ERROR, "Server response validating failed\n");
                return AVERROR(EIO);
            }
        }

        /* Generate SWFVerification token (SHA256 HMAC hash of decompressed SWF,
         * key are the last 32 bytes of the server handshake. */
        if (rt->swfsize) {
            if ((ret = rtmp_calc_swf_verification(s, rt, serverdata + 1 +
                                                  RTMP_HANDSHAKE_PACKET_SIZE - 32)) < 0)
                return ret;
        }

        ret = ff_rtmp_calc_digest(tosend + 1 + client_pos, 32, 0,
                                  rtmp_server_key, sizeof(rtmp_server_key),
                                  digest);
        if (ret < 0)
            return ret;

        ret = ff_rtmp_calc_digest(clientdata, RTMP_HANDSHAKE_PACKET_SIZE - 32,
                                  0, digest, 32, signature);
        if (ret < 0)
            return ret;

        if (CONFIG_FFRTMPCRYPT_PROTOCOL && rt->encrypted) {
            /* Compute the shared secret key sent by the server and initialize
             * the RC4 encryption. */
            if ((ret = ff_rtmpe_compute_secret_key(rt->stream, serverdata + 1,
                                                   tosend + 1, type)) < 0)
                return ret;

            /* Encrypt the signature received by the server. */
            ff_rtmpe_encrypt_sig(rt->stream, signature, digest, serverdata[0]);
        }

        if (memcmp(signature, clientdata + RTMP_HANDSHAKE_PACKET_SIZE - 32, 32)) {
            av_log(s, AV_LOG_ERROR, "Signature mismatch\n");
            return AVERROR(EIO);
        }

        for (i = 0; i < RTMP_HANDSHAKE_PACKET_SIZE; i++)
            tosend[i] = av_lfg_get(&rnd) >> 24;
        ret = ff_rtmp_calc_digest(serverdata + 1 + server_pos, 32, 0,
                                  rtmp_player_key, sizeof(rtmp_player_key),
                                  digest);
        if (ret < 0)
            return ret;

        /// Prepares C2 (1504 B random data + 32 B signature)
        ret = ff_rtmp_calc_digest(tosend, RTMP_HANDSHAKE_PACKET_SIZE - 32, 0,
                                  digest, 32,
                                  tosend + RTMP_HANDSHAKE_PACKET_SIZE - 32);
        if (ret < 0)
            return ret;

        if (CONFIG_FFRTMPCRYPT_PROTOCOL && rt->encrypted) {
            /* Encrypt the signature to be send to the server. */
            ff_rtmpe_encrypt_sig(rt->stream, tosend +
                                 RTMP_HANDSHAKE_PACKET_SIZE - 32, digest,
                                 serverdata[0]);
        }

        // write reply back to the server
        if ((ret = ffurl_write(rt->stream, tosend,
                               RTMP_HANDSHAKE_PACKET_SIZE)) < 0)
            return ret;

        if (CONFIG_FFRTMPCRYPT_PROTOCOL && rt->encrypted) {
            /* Set RC4 keys for encryption and update the keystreams. */
            if ((ret = ff_rtmpe_update_keystream(rt->stream)) < 0)
                return ret;
        }
    } else {
        if (CONFIG_FFRTMPCRYPT_PROTOCOL && rt->encrypted) {
            /* Compute the shared secret key sent by the server and initialize
             * the RC4 encryption. */
            if ((ret = ff_rtmpe_compute_secret_key(rt->stream, serverdata + 1,
                            tosend + 1, 1)) < 0)
                return ret;

            if (serverdata[0] == 9) {
                /* Encrypt the signature received by the server. */
                ff_rtmpe_encrypt_sig(rt->stream, signature, digest,
                                     serverdata[0]);
            }
        }

        /// If FFmpeg is the publisher, writes back a copy of S1 as C2.
        /// (!) I think this is incorrect.
        if ((ret = ffurl_write(rt->stream, serverdata + 1,
                               RTMP_HANDSHAKE_PACKET_SIZE)) < 0)
            return ret;

        if (CONFIG_FFRTMPCRYPT_PROTOCOL && rt->encrypted) {
            /* Set RC4 keys for encryption and update the keystreams. */
            if ((ret = ff_rtmpe_update_keystream(rt->stream)) < 0)
                return ret;
        }
    }

    return 0;
}

static int rtmp_receive_hs_packet(RTMPContext* rt, uint32_t *first_int,
                                  uint32_t *second_int, char *arraydata,
                                  int size)
{
    int inoutsize;

    inoutsize = ffurl_read_complete(rt->stream, arraydata,
                                    RTMP_HANDSHAKE_PACKET_SIZE);
    if (inoutsize <= 0)
        return AVERROR(EIO);
    if (inoutsize != RTMP_HANDSHAKE_PACKET_SIZE) {
        av_log(rt, AV_LOG_ERROR, "Erroneous Message size %d"
               " not following standard\n", (int)inoutsize);
        return AVERROR(EINVAL);
    }

    *first_int  = AV_RB32(arraydata);
    *second_int = AV_RB32(arraydata + 4);
    return 0;
}

/**
 * Put HMAC-SHA2 digest of packet data (except for the bytes where this digest
 * will be stored) into that packet.
 *
 * @param buf handshake data (1536 bytes)
 * @param encrypted use an encrypted connection (RTMPE)
 * @return offset to the digest inside input data
 */
static int rtmp_handshake_imprint_with_digest(uint8_t *buf, int encrypted)
{
    int ret, digest_pos;

    if (encrypted)
        /// For key-digest
        digest_pos = ff_rtmp_calc_digest_pos(buf, 772, 728, 776);
    else
        /// For digest-key
        digest_pos = ff_rtmp_calc_digest_pos(buf, 8, 728, 12);

    ret = ff_rtmp_calc_digest(buf, RTMP_HANDSHAKE_PACKET_SIZE, digest_pos,
                              rtmp_player_key, PLAYER_KEY_OPEN_PART_LEN,
                              buf + digest_pos);
    if (ret < 0)
        return ret;

    return digest_pos;
}

/**
 * Verify that the received server response has the expected digest value.
 *
 * @param buf handshake data received from the server (1536 bytes)
 * @param off position to search digest offset from
 * @return 0 if digest is valid, digest position otherwise
 */
static int rtmp_validate_digest(uint8_t *buf, int off)
{
    uint8_t digest[32];
    int ret, digest_pos;

    digest_pos = ff_rtmp_calc_digest_pos(buf, off, 728, off + 4);

    ret = ff_rtmp_calc_digest(buf, RTMP_HANDSHAKE_PACKET_SIZE, digest_pos,
                              rtmp_server_key, SERVER_KEY_OPEN_PART_LEN,
                              digest);
    if (ret < 0)
        return ret;

    if (!memcmp(digest, buf + digest_pos, 32))
        return digest_pos;
    return 0;
}


/****************************************************************
 *
 * From "libavformat/rtmpdigest.c"
 *
 ****************************************************************/


int ff_rtmp_calc_digest(const uint8_t *src, int len, int gap,
                        const uint8_t *key, int keylen, uint8_t *dst)
{
    AVHMAC *hmac;

    hmac = av_hmac_alloc(AV_HMAC_SHA256);
    if (!hmac)
        return AVERROR(ENOMEM);

    av_hmac_init(hmac, key, keylen);
    if (gap <= 0) {
        av_hmac_update(hmac, src, len);
    } else { //skip 32 bytes used for storing digest
        av_hmac_update(hmac, src, gap);
        av_hmac_update(hmac, src + gap + 32, len - gap - 32);
    }
    av_hmac_final(hmac, dst, 32);

    av_hmac_free(hmac);

    return 0;
}

int ff_rtmp_calc_digest_pos(const uint8_t *buf, int off, int mod_val,
                            int add_val)
{
    int i, digest_pos = 0;

    for (i = 0; i < 4; i++)
        digest_pos += buf[i + off];
    digest_pos = digest_pos % mod_val + add_val;

    return digest_pos;
}