AES 128 - encrypt/decrypt using Windows Crypto API

aes_crypt.cpp

#include <Windows.h>
#include <wincrypt.h>
#include <stdio.h>
#pragma comment(lib, "advapi32.lib")

#define AES_KEY_SIZE 16
#define IN_CHUNK_SIZE (AES_KEY_SIZE * 10) // a buffer must be a multiple of the key size
#define OUT_CHUNK_SIZE (IN_CHUNK_SIZE * 2) // an output buffer (for encryption) must be twice as big

//params: <input file> <output file> <is decrypt mode> <key>
int wmain(int argc, wchar_t *argv[])
{
    if (argc < 4) {
        printf("params: <input file> <output file> <is decrypt mode> [*key]\n");
        system("pause");
        return 0;
    }
    wchar_t *filename = argv[1];
    wchar_t *filename2 = argv[2];

    wchar_t default_key[] = L"3igcZhRdWq96m3GUmTAiv9";
    wchar_t *key_str = default_key;

    BOOL isDecrypt = FALSE;
    if (argv[3][0] > '0') {
        printf("Decrypt mode\n");
        isDecrypt = TRUE;
    }
    if (argc >= 5) {
        key_str = argv[4];
    }
    const size_t len = lstrlenW(key_str);
    const size_t key_size = len * sizeof(key_str[0]); // size in bytes

    printf("Key: %S\n", key_str);
    printf("Key len: %#x\n", len);
    printf("Key size: %#x\n", key_size);
    printf("Input File: %S\n", filename);
    printf("Output File: %S\n", filename2);
    printf("----\n");

    HANDLE hInpFile = CreateFileW(filename, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_FLAG_SEQUENTIAL_SCAN, NULL);
    if (hInpFile == INVALID_HANDLE_VALUE) {
        printf("Cannot open input file!\n");
        system("pause");
        return (-1);
    }
    HANDLE hOutFile = CreateFileW(filename2, GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL);
    if (hOutFile == INVALID_HANDLE_VALUE) {
        printf("Cannot open output file!\n");
        system("pause");
        return (-1);
    }

    if (isDecrypt) {
        printf("DECRYPTING\n");
    }
    else {
        printf("ENCRYPTING\n");
    }

    DWORD dwStatus = 0;
    BOOL bResult = FALSE;
    wchar_t info[] = L"Microsoft Enhanced RSA and AES Cryptographic Provider";
    HCRYPTPROV hProv;
    if (!CryptAcquireContextW(&hProv, NULL, info, PROV_RSA_AES, CRYPT_VERIFYCONTEXT)) {
        dwStatus = GetLastError();
        printf("CryptAcquireContext failed: %x\n", dwStatus);
        CryptReleaseContext(hProv, 0);
        system("pause");
        return dwStatus;
    }
    HCRYPTHASH hHash;
    if (!CryptCreateHash(hProv, CALG_SHA_256, 0, 0, &hHash)) {
        dwStatus = GetLastError();
        printf("CryptCreateHash failed: %x\n", dwStatus);
        CryptReleaseContext(hProv, 0);
        system("pause");
        return dwStatus;
    }

    if (!CryptHashData(hHash, (BYTE*)key_str, key_size, 0)) {
        DWORD err = GetLastError();
        printf("CryptHashData Failed : %#x\n", err);
        system("pause");
        return (-1);
    }
    printf("[+] CryptHashData Success\n");

    HCRYPTKEY hKey;
    if (!CryptDeriveKey(hProv, CALG_AES_128, hHash, 0, &hKey)) {
        dwStatus = GetLastError();
        printf("CryptDeriveKey failed: %x\n", dwStatus);
        CryptReleaseContext(hProv, 0);
        system("pause");
        return dwStatus;
    }
    printf("[+] CryptDeriveKey Success\n");

    const size_t chunk_size = isDecrypt ? IN_CHUNK_SIZE: OUT_CHUNK_SIZE;
    BYTE *chunk = new BYTE[chunk_size];
    DWORD out_len = 0;

    BOOL isFinal = FALSE;
    DWORD readTotalSize = 0;

    DWORD inputSize = GetFileSize(hInpFile, NULL);

    while (bResult = ReadFile(hInpFile, chunk, IN_CHUNK_SIZE, &out_len, NULL)) {
        if (0 == out_len) {
            break;
        }
        readTotalSize += out_len;
        if (readTotalSize >= inputSize) {
            isFinal = TRUE;
            printf("Final chunk set, len: %d = %x\n", out_len, out_len);
        }

        if (isDecrypt) {
            if (!CryptDecrypt(hKey, NULL, isFinal, 0, chunk, &out_len)) {
                printf("[-] CryptDecrypt failed: %x\n", GetLastError());
                break;
            }
        }
        else {
            if (!CryptEncrypt(hKey, NULL, isFinal, 0, chunk, &out_len, chunk_size)) {
                printf("[-] CryptEncrypt failed: %x\n", GetLastError());
                break;
            }
        }
        DWORD written = 0;
        if (!WriteFile(hOutFile, chunk, out_len, &written, NULL)) {
            printf("writing failed!\n");
            break;
        }
        memset(chunk, 0, chunk_size);
    }
    
    delete[]chunk; chunk = NULL;

    CryptDestroyHash(hHash);
    CryptDestroyKey(hKey);
    CryptReleaseContext(hProv, 0);

    CloseHandle(hInpFile);
    CloseHandle(hOutFile);
    printf("Finished. Processed %#x bytes.\n", readTotalSize);
    return 0;
}

Excuse me, @hasherezade . I'm very new to coding anything. I tried your code, and then found this:
[error: 'CALG_SHA_256' was not declared in this scope]
How do you fix this in the code? Again, very new to coding. Thx

In CryptDecrypt you're using NULL in the second parameter but according to the documentation it should be 0. (Compiling with gcc gives a warning about it...)

What should I put at "is decrypt mode" param?

What should I put at "is decrypt mode" param?

"is decrypt mode" is a flag that switches between decryption and encryption. if it is set to 0, the supplied file will be encrypted, otherwise it will be decrypted.

Wrong order for releasing
right order:

    CryptDestroyHash(hHash);
    CryptDestroyKey(hKey);
    CryptReleaseContext(hProv, 0);

ref: https://docs.microsoft.com/en-us/windows/win32/seccrypto/example-c-program-encrypting-a-file

Very nice source and example, but I noticed a flaw - The user-supplied key is treated as a wchar_t, and its length is calculated appropriately by lstrlenW, but CryptHashData treats its argument as a byte array and treats the len parameter as the number of bytes (https://docs.microsoft.com/en-us/windows/win32/api/wincrypt/nf-wincrypt-crypthashdata), thus hashes only half of the key.
For example, if the user typed abcd, the actual data in memory is a\x00b\x00c\x00d\x00 with length 4 from lstrlenW, but CryptHashData gets passed 4 as the number of bytes, and will only hash a\x00b\x00.
This weakens the security of the encryption, as it halves the number of bits needed to bruteforce the user-supplied key (exponential factor).

@YonPog - you are right, thank you for noticing it! fixed

It works fine but it didn't work with RSA key pair. Any ideas?

maybe it will be useful to someone: an example of decryption in python:

import hashlib
from Crypto.Cipher import AES

def hash_md5(data):
    md5_obj = hashlib.md5()
    md5_obj.update(data)
    return md5_obj.digest()

def mscrypt_derive_key_md5(password):
    buf1 = bytearray([0x36] * 64)
    buf2 = bytearray([0x5C] * 64)

    hash_obj = hashlib.md5()
    hash_obj.update(password)
    hash_result = hash_obj.digest()

    buf1[:len(hash_result)] = [buf1[i] ^ hash_result[i] for i in range(len(hash_result))]
    buf2[:len(hash_result)] = [buf2[i] ^ hash_result[i] for i in range(len(hash_result))]

    hash_buf1 = hash_md5(buf1)
    hash_buf2 = hash_md5(buf2)
    
    return hash_buf1 + hash_buf2

input_file = r'D:\enc.dat'
output_file = r'D:\dec.dat'

with open(input_file, 'rb') as f:
    encrypted_data = f.read()

base_password = "3igcZhRdWq96m3GUmTAiv9"
my_derived_key = mscrypt_derive_key_md5(base_password.encode('utf-8'))

print("My Derived Key:\t\t", my_derived_key.hex())

cipher = AES.new(my_derived_key, AES.MODE_CBC, iv=b'\x00'*16)
decrypted_data = cipher.decrypt(encrypted_data)

with open(output_file, 'wb') as f:
    f.write(decrypted_data)

P.S in example used md5 hash.