Annotated PK11_ExportEncryptedPrivKeyInfo - with https://github.com/cipherboy/nss/pull/6 (Notes prefixed with AS:)

pk11akey.c

SECKEYEncryptedPrivateKeyInfo *
PK11_ExportEncryptedPrivKeyInfo(
    PK11SlotInfo *slot,   /* optional, encrypt key in this slot */
    SECOidTag algTag,     /* encrypt key with this algorithm
                             AS: this needs to be updated to the SEC_OID_AES_KEY_WRAP_PAD.
                                 requires changing JSS. */
    SECItem *pwitem,      /* password for PBE encryption */
    SECKEYPrivateKey *pk, /* encrypt this private key */
    int iteration,        /* interations for PBE alg */
    void *wincx)          /* context for password callback ? */
{
    SECKEYEncryptedPrivateKeyInfo *epki = NULL;
    PLArenaPool *arena = NULL;
    SECAlgorithmID *algid;
    SECOidTag pbeAlgTag = SEC_OID_UNKNOWN;
    SECItem *crypto_param = NULL;
    PK11SymKey *key = NULL;
    SECKEYPrivateKey *tmpPK = NULL;
    SECStatus rv = SECSuccess;
    CK_RV crv;
    CK_ULONG encBufLen;
    CK_MECHANISM_TYPE pbeMechType;
    CK_MECHANISM_TYPE cryptoMechType;
    CK_MECHANISM cryptoMech;

    if (!pwitem || !pk) {
        PORT_SetError(SEC_ERROR_INVALID_ARGS);
        return NULL;
    }

    algid = sec_pkcs5CreateAlgorithmID(algTag, SEC_OID_UNKNOWN, SEC_OID_UNKNOWN,
                                       &pbeAlgTag, 0, NULL, iteration);
    if (algid == NULL) {
        return NULL;
    }

    arena = PORT_NewArena(2048);
    if (arena)
        epki = PORT_ArenaZNew(arena, SECKEYEncryptedPrivateKeyInfo);
    if (epki == NULL) {
        rv = SECFailure;
        goto loser;
    }
    epki->arena = arena;

    /* if we didn't specify a slot, use the slot the private key was in */
    if (!slot) {
        slot = pk->pkcs11Slot;
    }

    /* if we specified a different slot, and the private key slot can do the
     * pbe key gen, generate the key in the private key slot so we don't have
     * to move it later */
    pbeMechType = PK11_AlgtagToMechanism(pbeAlgTag);
    if (slot != pk->pkcs11Slot) {
        if (PK11_DoesMechanism(pk->pkcs11Slot, pbeMechType)) {
            slot = pk->pkcs11Slot;
        }
    }
  
    /* AS: This actually does the PBEKeyGen. Here, from testing, we used the internal
           slot. The output is a standard AES key, so with the logic below, it is portable to
           the external/HSM token. The HSM will then encrypt its private key without the private
           key leaving the HSM and without the HSM needing knowledge of PB2 algorithm mechanisms.
           To the HSM, it just looks like an RSA wrap+unwrap coupled with a AES encrypt operation.
           The subtle lies in the flags. */
    key = PK11_PBEKeyGen(slot, algid, pwitem, PR_FALSE, wincx); 
    if (key == NULL) {
        rv = SECFailure;
        goto loser;
    }

     /* AS: If algTag isn't set properly above, this is where an invalid type gets used. */
    cryptoMechType = PK11_GetPBECryptoMechanism(algid, &crypto_param, pwitem);
    if (cryptoMechType == CKM_INVALID_MECHANISM) {
        rv = SECFailure;
        goto loser;
    }

    cryptoMech.mechanism = PK11_GetPadMechanism(cryptoMechType);
    cryptoMech.pParameter = crypto_param ? crypto_param->data : NULL;
    cryptoMech.ulParameterLen = crypto_param ? crypto_param->len : 0;

    /* If the key isn't in the private key slot, move it */
    if (key->slot != pk->pkcs11Slot) {
        /* AS: Here in my diff, I changed the key->type to the cryptoMech.mechanism.
               This was intentional. key->type claims the keytype to be PB2 something.
               While strictly true, no HSM supports that. And, the actual operation
               we want to do with the key is cryptoMech.mechanism. So when trying to
               import the key to the HSM, pretend that it is just an AES key, which is
               all the new slot will be using it for. It doesn't need to know about its
               past history.
               
               This is what actually does the wrap/unwrap dance on the HSM to make
               exporting the key possible. Note that all this does is copy our new
               temporary AES key to the HSM using any existing RSA key already on the
               HSM with a workable private key. */
        PK11SymKey *newkey = pk11_CopyToSlot(pk->pkcs11Slot,
                                             cryptoMech.mechanism, CKA_WRAP, key);
        if (newkey == NULL) {
            /* couldn't import the wrapping key, try exporting the
             * private key */
            /* AS: path not taken. Above should've worked in the HSM case. */
            tmpPK = pk11_loadPrivKey(key->slot, pk, NULL, PR_FALSE, PR_TRUE);
            if (tmpPK == NULL) {
                rv = SECFailure;
                goto loser;
            }
            pk = tmpPK;
        } else {
            /* free the old key and use the new key */
            PK11_FreeSymKey(key);
            key = newkey;
        }
    }

    /* we are extracting an encrypted privateKey structure.
     * which needs to be freed along with the buffer into which it is
     * returned.  eventually, we should retrieve an encrypted key using
     * pkcs8/pkcs5.
     */
  
    
    /* AS: Our new AES key (created by PB2 mech) should now reside on the HSM.
           All we need to do now is wrap the right information from the HSM and
           spit it out in a PKCS12-ish blob. Since the HSM should support
           cryptoMech.mechanism, this should be possible and the HSM will be
           none-the-wiser that it is actually doing PKCS12 and not wrapping as
           part of some more secure protocol. */
    encBufLen = 0;
    PK11_EnterSlotMonitor(pk->pkcs11Slot);
    crv = PK11_GETTAB(pk->pkcs11Slot)->C_WrapKey(pk->pkcs11Slot->session, &cryptoMech, key->objectID, pk->pkcs11ID, NULL, &encBufLen);
    PK11_ExitSlotMonitor(pk->pkcs11Slot);
    if (crv != CKR_OK) {
        rv = SECFailure;
        goto loser;
    }
    epki->encryptedData.data = PORT_ArenaAlloc(arena, encBufLen);
    if (!epki->encryptedData.data) {
        rv = SECFailure;
        goto loser;
    }
    PK11_EnterSlotMonitor(pk->pkcs11Slot);
    crv = PK11_GETTAB(pk->pkcs11Slot)->C_WrapKey(pk->pkcs11Slot->session, &cryptoMech, key->objectID, pk->pkcs11ID, epki->encryptedData.data, &encBufLen);
    PK11_ExitSlotMonitor(pk->pkcs11Slot);
    epki->encryptedData.len = (unsigned int)encBufLen;
    if (crv != CKR_OK) {
        rv = SECFailure;
        goto loser;
    }

    if (!epki->encryptedData.len) {
        rv = SECFailure;
        goto loser;
    }

    rv = SECOID_CopyAlgorithmID(arena, &epki->algorithm, algid);

loser:
    if (crypto_param != NULL) {
        SECITEM_ZfreeItem(crypto_param, PR_TRUE);
        crypto_param = NULL;
    }

    if (key != NULL) {
        PK11_FreeSymKey(key);
    }
    if (tmpPK != NULL) {
        SECKEY_DestroyPrivateKey(tmpPK);
    }
    SECOID_DestroyAlgorithmID(algid, PR_TRUE);

    if (rv == SECFailure) {
        if (arena != NULL) {
            PORT_FreeArena(arena, PR_TRUE);
        }
        epki = NULL;
    }

    return epki;
}