AES 128 - encrypt/decrypt using Windows Crypto API

aes_crypt.cpp

#include <Windows.h>
#include <wincrypt.h>
#include <stdio.h>
#pragma comment(lib, "advapi32.lib")

#define AES_KEY_SIZE 16
#define IN_CHUNK_SIZE (AES_KEY_SIZE * 10) // a buffer must be a multiple of the key size
#define OUT_CHUNK_SIZE (IN_CHUNK_SIZE * 2) // an output buffer (for encryption) must be twice as big

//params: <input file> <output file> <is decrypt mode> <key>
int wmain(int argc, wchar_t *argv[])
{
    if (argc < 4) {
        printf("params: <input file> <output file> <is decrypt mode> [*key]\n");
        system("pause");
        return 0;
    }
    wchar_t *filename = argv[1];
    wchar_t *filename2 = argv[2];

    wchar_t default_key[] = L"3igcZhRdWq96m3GUmTAiv9";
    wchar_t *key_str = default_key;

    BOOL isDecrypt = FALSE;
    if (argv[3][0] > '0') {
        printf("Decrypt mode\n");
        isDecrypt = TRUE;
    }
    if (argc >= 5) {
        key_str = argv[4];
    }
    const size_t len = lstrlenW(key_str);
    const size_t key_size = len * sizeof(key_str[0]); // size in bytes

    printf("Key: %S\n", key_str);
    printf("Key len: %#x\n", len);
    printf("Key size: %#x\n", key_size);
    printf("Input File: %S\n", filename);
    printf("Output File: %S\n", filename2);
    printf("----\n");

    HANDLE hInpFile = CreateFileW(filename, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_FLAG_SEQUENTIAL_SCAN, NULL);
    if (hInpFile == INVALID_HANDLE_VALUE) {
        printf("Cannot open input file!\n");
        system("pause");
        return (-1);
    }
    HANDLE hOutFile = CreateFileW(filename2, GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL);
    if (hOutFile == INVALID_HANDLE_VALUE) {
        printf("Cannot open output file!\n");
        system("pause");
        return (-1);
    }

    if (isDecrypt) {
        printf("DECRYPTING\n");
    }
    else {
        printf("ENCRYPTING\n");
    }

    DWORD dwStatus = 0;
    BOOL bResult = FALSE;
    wchar_t info[] = L"Microsoft Enhanced RSA and AES Cryptographic Provider";
    HCRYPTPROV hProv;
    if (!CryptAcquireContextW(&hProv, NULL, info, PROV_RSA_AES, CRYPT_VERIFYCONTEXT)) {
        dwStatus = GetLastError();
        printf("CryptAcquireContext failed: %x\n", dwStatus);
        CryptReleaseContext(hProv, 0);
        system("pause");
        return dwStatus;
    }
    HCRYPTHASH hHash;
    if (!CryptCreateHash(hProv, CALG_SHA_256, 0, 0, &hHash)) {
        dwStatus = GetLastError();
        printf("CryptCreateHash failed: %x\n", dwStatus);
        CryptReleaseContext(hProv, 0);
        system("pause");
        return dwStatus;
    }

    if (!CryptHashData(hHash, (BYTE*)key_str, key_size, 0)) {
        DWORD err = GetLastError();
        printf("CryptHashData Failed : %#x\n", err);
        system("pause");
        return (-1);
    }
    printf("[+] CryptHashData Success\n");

    HCRYPTKEY hKey;
    if (!CryptDeriveKey(hProv, CALG_AES_128, hHash, 0, &hKey)) {
        dwStatus = GetLastError();
        printf("CryptDeriveKey failed: %x\n", dwStatus);
        CryptReleaseContext(hProv, 0);
        system("pause");
        return dwStatus;
    }
    printf("[+] CryptDeriveKey Success\n");

    const size_t chunk_size = isDecrypt ? IN_CHUNK_SIZE: OUT_CHUNK_SIZE;
    BYTE *chunk = new BYTE[chunk_size];
    DWORD out_len = 0;

    BOOL isFinal = FALSE;
    DWORD readTotalSize = 0;

    DWORD inputSize = GetFileSize(hInpFile, NULL);

    while (bResult = ReadFile(hInpFile, chunk, IN_CHUNK_SIZE, &out_len, NULL)) {
        if (0 == out_len) {
            break;
        }
        readTotalSize += out_len;
        if (readTotalSize >= inputSize) {
            isFinal = TRUE;
            printf("Final chunk set, len: %d = %x\n", out_len, out_len);
        }

        if (isDecrypt) {
            if (!CryptDecrypt(hKey, NULL, isFinal, 0, chunk, &out_len)) {
                printf("[-] CryptDecrypt failed: %x\n", GetLastError());
                break;
            }
        }
        else {
            if (!CryptEncrypt(hKey, NULL, isFinal, 0, chunk, &out_len, chunk_size)) {
                printf("[-] CryptEncrypt failed: %x\n", GetLastError());
                break;
            }
        }
        DWORD written = 0;
        if (!WriteFile(hOutFile, chunk, out_len, &written, NULL)) {
            printf("writing failed!\n");
            break;
        }
        memset(chunk, 0, chunk_size);
    }
    
    delete[]chunk; chunk = NULL;

    CryptDestroyHash(hHash);
    CryptDestroyKey(hKey);
    CryptReleaseContext(hProv, 0);

    CloseHandle(hInpFile);
    CloseHandle(hOutFile);
    printf("Finished. Processed %#x bytes.\n", readTotalSize);
    return 0;
}

it didn't work for me !
i get the empty file with 0kb size

Gives Error 234, which boils down to,
If the buffer allocated for pbData is not large enough to hold the data, GetLastError returns ERROR_MORE_DATA and stores the required buffer size, in bytes, in the DWORD value pointed to by pdwDataLen.

either delete this gist or correct it.
nice work on malwares though. 💃

Wow, I didn't knew that someone is using it. It's my old, long-forgotten note. I will fix it.

@0x410c - can you elaborate on the error that you was getting? does it persist in the new version?

You are using a block size of 128 bytes when it is really 128 bits or 16 bytes. Not sure how your implementation impacts the security of the encrypted data but I am sure there must be issues.

yo this is a really nice source. I got this to work on both visual studio 6.0 (1998) and visual studio 2017 /w 10 sdk...
I have tried many MANY sources/tutorials/demos/samples/ even MSDN documentation and none of it worked, this worked and not only did it work does so in only 142 lines of code! I had the original AES Rijndael code its someting like 1379 lines and had to manually change strings to bytes to hex and do the padding.. but this is very very clean. once again thankyou so much Your other sources are pretty beast as well (NTDDK is new for me) and I am learning.

Excuse me, @hasherezade . I'm very new to coding anything. I tried your code, and then found this:
[error: 'CALG_SHA_256' was not declared in this scope]
How do you fix this in the code? Again, very new to coding. Thx

In CryptDecrypt you're using NULL in the second parameter but according to the documentation it should be 0. (Compiling with gcc gives a warning about it...)

What should I put at "is decrypt mode" param?

What should I put at "is decrypt mode" param?

"is decrypt mode" is a flag that switches between decryption and encryption. if it is set to 0, the supplied file will be encrypted, otherwise it will be decrypted.

Wrong order for releasing
right order:

    CryptDestroyHash(hHash);
    CryptDestroyKey(hKey);
    CryptReleaseContext(hProv, 0);

ref: https://docs.microsoft.com/en-us/windows/win32/seccrypto/example-c-program-encrypting-a-file

Very nice source and example, but I noticed a flaw - The user-supplied key is treated as a wchar_t, and its length is calculated appropriately by lstrlenW, but CryptHashData treats its argument as a byte array and treats the len parameter as the number of bytes (https://docs.microsoft.com/en-us/windows/win32/api/wincrypt/nf-wincrypt-crypthashdata), thus hashes only half of the key.
For example, if the user typed abcd, the actual data in memory is a\x00b\x00c\x00d\x00 with length 4 from lstrlenW, but CryptHashData gets passed 4 as the number of bytes, and will only hash a\x00b\x00.
This weakens the security of the encryption, as it halves the number of bits needed to bruteforce the user-supplied key (exponential factor).

@YonPog - you are right, thank you for noticing it! fixed

It works fine but it didn't work with RSA key pair. Any ideas?

maybe it will be useful to someone: an example of decryption in python:

import hashlib
from Crypto.Cipher import AES

def hash_md5(data):
    md5_obj = hashlib.md5()
    md5_obj.update(data)
    return md5_obj.digest()

def mscrypt_derive_key_md5(password):
    buf1 = bytearray([0x36] * 64)
    buf2 = bytearray([0x5C] * 64)

    hash_obj = hashlib.md5()
    hash_obj.update(password)
    hash_result = hash_obj.digest()

    buf1[:len(hash_result)] = [buf1[i] ^ hash_result[i] for i in range(len(hash_result))]
    buf2[:len(hash_result)] = [buf2[i] ^ hash_result[i] for i in range(len(hash_result))]

    hash_buf1 = hash_md5(buf1)
    hash_buf2 = hash_md5(buf2)
    
    return hash_buf1 + hash_buf2

input_file = r'D:\enc.dat'
output_file = r'D:\dec.dat'

with open(input_file, 'rb') as f:
    encrypted_data = f.read()

base_password = "3igcZhRdWq96m3GUmTAiv9"
my_derived_key = mscrypt_derive_key_md5(base_password.encode('utf-8'))

print("My Derived Key:\t\t", my_derived_key.hex())

cipher = AES.new(my_derived_key, AES.MODE_CBC, iv=b'\x00'*16)
decrypted_data = cipher.decrypt(encrypted_data)

with open(output_file, 'wb') as f:
    f.write(decrypted_data)

P.S in example used md5 hash.