Playing with AGE (actually good encryption) to try some test vectors from ツ.

README.md

This repository contains my simplified AGE (Actually Good Encryption) http://age-encryption.org/v1 heavily based on pyage library.

It's purpose is to demonstrate the fact that GRIN core wallet implementation is producing incorrect HMAC signatures in its AGE-encrypted payloads.

1. slateage.py contains values implementated by the GRIN core wallet. My code correctly decrypts it only if HMAC verification is ignored. Decrypted payload contains the sender address.
2. validage.py contains official AGE test vector. My code correctly decrypts it and verifies the HMAC and matches the expected file key. Decrypted payload hash matches the expected value.

Another indicator the GRIN core wallet is producing flawed HMAC signatures is the fact that in grin++ wallet the HMAC verification also had to be commented out.

To run it, first setup

pip install -r requirements.txt

Output of slateage.py

sender  : grin1m4krnajw792zxfyldu79jssh0d3kzjtwpdn2wy7fysrfw4ej0waskurq76
receiver: grin14kgku7l5x6te3arast3p59zk4rteznq2ug6kmmypf2d6z8md76eqg3su35

Slatepack info:
Checksum valid: True
Version major : 1
Version minor : 0
Emode         : 1

AGE decryption succeeded.

Sender address grin1m4krnajw792zxfyldu79jssh0d3kzjtwpdn2wy7fysrfw4ej0waskurq76 found in the decrypted payload.
AGE payload correctly decrypted.

Output of validage.py

decrypted payload is
b'age'

helpers.py

import base58
import hashlib

from bip32 import BIP32
from bip_utils import Bech32Encoder
from nacl import bindings
from io import BytesIO

SLATEPACK_HEADER = 'BEGINSLATEPACK'
SLATEPACK_FOOTER = 'ENDSLATEPACK'

def slatepack_unpack(data: str) -> bytes:
    processed = data.replace(SLATEPACK_HEADER, '')
    processed = processed.replace(SLATEPACK_FOOTER, '')
    processed = processed.replace('.', '')
    processed = processed.replace(' ', '')
    processed = processed.replace('\n', '')
    return base58.b58decode(processed)

def slatepack_checksum(data: bytes):
    b = BytesIO()
    b.write(data)
    b.seek(0)
    b.read(4)
    preimage = b.read()
    hashed = hashlib.sha256(preimage).digest()
    hashed = hashlib.sha256(hashed).digest()
    return hashed[0:4]

def slatepack_to_age(data: bytes):
    b = BytesIO()
    b.write(data)
    b.seek(0)

    error_check_code = b.read(4)
    computed_checksum = slatepack_checksum(data)

    valid = error_check_code == computed_checksum

    version_major = int.from_bytes(b.read(1), 'big')
    version_minor = int.from_bytes(b.read(1), 'big')

    emode = int.from_bytes(b.read(1), 'big')

    opt_flags = int.from_bytes(b.read(2), 'big')
    opt_fields_len = int.from_bytes(b.read(4), 'big')

    address = None
    if opt_flags & 0x01 == 0x01:
        address_length = int.from_bytes(b.read(1), 'big')
        address = b.read(address_length).decode('ascii')

    payload_size = int.from_bytes(b.read(8), 'big')
    payload = b.read(payload_size)

    return valid, version_major, version_minor, emode, address, payload


def grin_wallet(master_key: bytes, path: str):
    bip32 = BIP32(
        chaincode=master_key[32:], privkey=master_key[:32])
    sk = bip32.get_privkey_from_path(path)

    seed_blake = hashlib.blake2b(sk, digest_size=32).digest()

    ed25519pk, ed25519sk = bindings.crypto_sign_seed_keypair(seed_blake)

    x25519pk = bindings.crypto_sign_ed25519_pk_to_curve25519(ed25519pk)
    x25519sk = bindings.crypto_sign_ed25519_sk_to_curve25519(ed25519sk)

    slatepack_address = Bech32Encoder.Encode('grin', ed25519pk)

    return ed25519pk, ed25519sk, x25519pk, x25519sk, slatepack_address

myage.py

import base64
import math

from io import BytesIO

from nacl.bindings import crypto_scalarmult
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import hashes
from cryptography.hazmat.primitives.kdf.hkdf import HKDF
from cryptography.hazmat.primitives.ciphers.aead import ChaCha20Poly1305
from cryptography.hazmat.primitives.hmac import HMAC


def b64decode(value):
    try:
        return base64.b64decode(value)
    except Exception as e:
        if 'Incorrect padding' in str(e):
            return base64.b64decode(value + b'===')
        raise e


def age_decrypt(
        mac_message,
        ephemeral_share,
        encrypted_file_key,
        header_mac,
        encrypted_payload,
        x25519pk,
        x25519sk, ignore_hmac=False,
        expected_decrypted_file_key=None):
    decoded_ephemeral_share = b64decode(ephemeral_share)
    decoded_encrypted_file_key = b64decode(encrypted_file_key)
    decoded_header_mac = b64decode(header_mac)

    public_key = x25519pk
    private_key = x25519sk

    salt = decoded_ephemeral_share + public_key
    key_material = crypto_scalarmult(
        private_key, decoded_ephemeral_share)

    AGE_X25519_HKDF_LABEL = b'age-encryption.org/v1/X25519'

    key = HKDF(
        algorithm=hashes.SHA256(),
        length=32,
        salt=salt,
        info=AGE_X25519_HKDF_LABEL,
        backend=default_backend()
    ).derive(key_material)

    ZERO_NONCE = b'\00' * 12

    decrypted_file_key = ChaCha20Poly1305(key).decrypt(
        ZERO_NONCE,
        decoded_encrypted_file_key,
        None)

    if expected_decrypted_file_key is not None:
        assert expected_decrypted_file_key == decrypted_file_key

    HEADER_HKDF_LABEL = b'header'
    salt = b''
    hkdfval = HKDF(
        algorithm=hashes.SHA256(),
        length=32,
        salt=salt,
        info=HEADER_HKDF_LABEL,
        backend=default_backend()
    ).derive(decrypted_file_key)

    hmac = HMAC(
        key=hkdfval,
        algorithm=hashes.SHA256(),
        backend=default_backend()
    )
    hmac.update(mac_message)
    if not ignore_hmac:
        hmac.verify(decoded_header_mac)

    stream = BytesIO()
    stream.write(encrypted_payload)
    stream.seek(0)

    nonce = stream.read(16)

    PAYLOAD_HKDF_LABEL = b'payload'
    stream_key = HKDF(
        algorithm=hashes.SHA256(),
        length=32,
        salt=nonce,
        info=PAYLOAD_HKDF_LABEL,
        backend=default_backend()
    ).derive(decrypted_file_key)

    ciphertext = stream.read()

    plaintext = b''

    PLAINTEXT_BLOCK_SIZE = 64 * 1024
    CIPHERTEXT_BLOCK_SIZE = PLAINTEXT_BLOCK_SIZE + 16

    aead = ChaCha20Poly1305(stream_key)

    blocks = math.ceil(len(ciphertext) / CIPHERTEXT_BLOCK_SIZE)

    def _chunk(data, size):
        for i in range(0, len(data), size):
            yield data[i : i + size]

    for nonce, block in enumerate(_chunk(ciphertext, CIPHERTEXT_BLOCK_SIZE)):
        last_block = nonce == blocks - 1
        packed_nonce = nonce.to_bytes(
            11,
            byteorder='big',
            signed=False
        ) + (b'\x01' if last_block else b'\x00')

        decrypted_chunk = aead.decrypt(
            nonce=packed_nonce,
            data=block,
            associated_data=None)
        plaintext += decrypted_chunk

    return plaintext

requirements.txt

base58
bip32
bip_utils
cryptography
pynacl

slateage.py

from myage import age_decrypt
from helpers import slatepack_unpack, slatepack_to_age, grin_wallet

# sender data
sender_master_key = '9a92a60868275955538c424eb8684d4e9f7d898478ed7f6799da8730f475b911b3dae82dedab5898485704303984f569b89b3700792da37de8851675c9ab0570'
_, _, _, _, sender = grin_wallet(
    bytes.fromhex(sender_master_key), 'm/0/1/0')
assert sender == 'grin1m4krnajw792zxfyldu79jssh0d3kzjtwpdn2wy7fysrfw4ej0waskurq76'

# receiver data
receiver_master_key = '31513630b8433beee67aea586e9b62fcd01258b1e5f75400e9ea58b065a496381d5d7accd414227b3e3c338884b91ac8ae0453015d37ac50190d1be75d1c72fb'
ed25519pk, ed25519sk, x25519pk, x25519sk, receiver = grin_wallet(
    bytes.fromhex(receiver_master_key), 'm/0/1/0')

assert x25519pk.hex() == '585b020389181ac540a9e268eb5f5d7d86297bad2ee0c8fb5eb533f92ce66d4d'
assert x25519sk.hex() == '90ad4d97dd2f7dd5360375f2ad72011489baa8beb84f1109b2bdb79b4183476a'
assert receiver == 'grin14kgku7l5x6te3arast3p59zk4rteznq2ug6kmmypf2d6z8md76eqg3su35'

print()
print('sender  :', sender)
print('receiver:', receiver)

slatepack = '''
BEGINSLATEPACK. auBNnRrahGnJ1iw 52RSppvCNzAPyT8 p5icMiMDYjDKbHH
8gd9Xci3AWGMd88 PWt36uc7uPVKocB SnxB28ptvgmfEn3 SouRUUBnjSEQCGi
gkwuzswEKLict2X A7sc7Rdu21gMFec Eq5AmyExTCjPHYg CU1DWQZC28kab8y
Fu1meQA5sYUQWM7 rvg1yADen6Z8R4S b3eVPg54eYwNv17 XqV1Lc3ACLSHycK
Gc7dPmAmBeZ7RxY JLdteR1QtFu8ngu GHSTNrui3TVkKug QJuN34WsJcCZWFc
AYKSYdBnwdXSPYy LsPCS3n4Mqo52HP U8kCq7sHsBdBbjV 9dcFQrm18pvWxVR
GJNm8XSrQtK9dyQ JvZxjv7UNTvh8q1 5yDXLA7z8L6NV2m dHZ6ujtecsSZdF5
mZqZyjsxeoj9kDr jjAXPD6gTVjobkh sjxXb1YU4qEfHnR wx7NjBx5RamzgEa
uWvARddnJd8pG2m wsptfQkvfBKogS5 1vRvmFMUb8MwPjW hucAnKcMaFLj1Hg
ESbV3HycopcL2VJ HJgmQQeFsqbyGMm Xkiz4sH2X6hWj1A D6rkR7uhDLL5YbY
MPwkFsRNK8zcPk7 X2DMCFZd5VNGcMZ gPBidMhw4nbUzii bj42vtLpT68JpTM
qeLUsuCzBUPs7h5 Z8vH9humdjkxkPM JK2z91cLcyWpqfN PCK2C96aWRmw1zK
vhA2bdauCkTRDD4 dES9RdcExqtMjby p2wvUFk2V2UHEw7 Sjpcc37kJ2P2ZG2
WYyM3VSXuMPSdwZ HVyvnXs4tSJTzsE 2wJTS5gRJX74FXW izjtA8tUGWcmCif
kBYie74pEWBAe4t ja5SLKkX4Ut1Ys8 rwPyB5zf6sGzrb7 3VyDtX85AmF7moE
MbaYHdP1tbM. ENDSLATEPACK.
'''

unpacked = slatepack_unpack(slatepack)
valid, version_major, version_minor, emode, address, age_payload = slatepack_to_age(
    unpacked)

print()
print('Slatepack info:')
print('Checksum valid:', valid)
print('Version major :', version_major)
print('Version minor :', version_minor)
print('Emode         :', emode)

ephemeral_share = b'F/H3hf63Hrq2MH+O1naWm6+M8184u3S7oeHQ4VHiMzY'
encrypted_file_key = b'pbUcLLHdT5PGn/oYeiD+tsUO1oexurSZmSVu8Hcv7mc'
header_mac = b'nq0hJv/bV3kLRrqKU9wlkexF2STZDmBf0hlees/HyWw'

mac_message = b'age-encryption.org/v1\n-> X25519 F/H3hf63Hrq2MH+O1naWm6+M8184u3S7oeHQ4VHiMzY\npbUcLLHdT5PGn/oYeiD+tsUO1oexurSZmSVu8Hcv7mc\n-> Es[-grease )vK1eC lVxuS`,6\nSkmgyCDG0wPb0wvGeYp6XbdSyMONfXYoRPLyQSlkse8tDxkTHKwjoz6Y3t2IIqHv\nU/dxyW2iZyZ1UVFRrlNAVbXbEBwqhpaFjDdRcivhbDo1uTL5CDNvJz89w+ivaoUc\n\n---'

reconstructed_mac_message = b'age-encryption.org/v1\n-> X25519 ' + ephemeral_share + b'\n' + encrypted_file_key + b'\n-> Es[-grease )vK1eC lVxuS`,6\nSkmgyCDG0wPb0wvGeYp6XbdSyMONfXYoRPLyQSlkse8tDxkTHKwjoz6Y3t2IIqHv\nU/dxyW2iZyZ1UVFRrlNAVbXbEBwqhpaFjDdRcivhbDo1uTL5CDNvJz89w+ivaoUc\n\n---'
assert reconstructed_mac_message == mac_message

encrypted_payload = b'\x8cA\xc3+\x1a\xaf\x10\x93T\x18\x0f\xf9J\x14\xaf\xb0\xcdw\xb9\x10\xa7\x90>V-\xc3=D\xafwc\xdeo\x08\xefx\xdd\x19\x85DtG\xad\xc8\x88*\x84\xe5Z\x07G\x95\x08\xad\xca\xb8\x81N\x1aJ\xfc\x97z\xa8\xfcR\xd8Y\xa4\x90\xcc[\x05\x989\xb1\x9d\xd1U\xdcsJu\xc7\xc4\xb9\x91\x03\x07\x08C\xd5\xfb>s\x80"\t\x85)\x8c(\x1dL\x9f\xa2\xea\xae\xf2f\xf8~\xfe\xc8\xc8\x14\xdeKD\xd8\x8b2\xd2\xca\xf4\xbdM\xcf\xe8_\x88\x03$=\xe1\x11(5g\xde\x9cF,S\xec0#*.\xa0\xe6H\x0f\x94\x8b\x02\xd1\xd9v\xea\xb8u%Dp\xb4\xb9\x88Jvz\xb9\x07\x1e$\x0f\xf5\x80\xe5\xbc\xe5I\x8b\x9f\xc8\xe1\x04\xec0T\x99\xf1cC\xd5\x90\xb27\x18\x1d\x88\x9d\x85\xea\x07\x11\xc4\x92\x0c\x07\xee\xc5\xb8\x1f\xf2&\xf29\x00R\xd4\xb6\xbfe\x18\xfd\xdd\x1a\x8f\xf0\xc3$\xfc0\xf3\xdc`\xed\xd0\xbd\xaa\xbd\t\x9d\xf1\x93.\xa6\xa6C\x0f\x0b;\x9er~P\xbd\xe1\xd5\xf5\x1f\xda>}\xe5J\x93\xc8\xde\xf4^\x9f\xbd\xbblK\xf3\xe3hP\xef\xdc\xb2\xbd\x05\xf1\x08\x9e\xe0\x96\xaeg\xd9\x1b\xc7\xf2/\xb8\xbe%aY\xdf\xc6T'

reconstructed = mac_message + b' ' + header_mac + b'\n' + encrypted_payload
assert reconstructed == age_payload

print()
try:
    decrypted_payload = age_decrypt(
        mac_message,
        ephemeral_share,
        encrypted_file_key,
        header_mac,
        encrypted_payload,
        x25519pk,
        x25519sk, ignore_hmac=False)
except Exception as e:
    print('AGE decryption failed due:', e)
    print('Attempting again, this time ignoring HMAC.')
    decrypted_payload = age_decrypt(
        mac_message,
        ephemeral_share,
        encrypted_file_key,
        header_mac,
        encrypted_payload,
        x25519pk,
        x25519sk, ignore_hmac=True)
print('AGE decryption succeeded.')

expected_decrypted_payload = b'\x00\x00\x00B\x00\x01?grin1m4krnajw792zxfyldu79jssh0d3kzjtwpdn2wy7fysrfw4ej0waskurq76\x00\x04\x00\x03S\xfe\x05\xcd\x07\x8bK\x8f\x96\x0bV\xafo\x1a3\xb1\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x00\x00\x00:\xdc\r\xe0\x00\x00\x00\x00\x00\xbe\xbc \x01\x00\x02S3\xf6~\x84^\xdbV\xe3\t\xd6\x9c\x1b\xc5k\x89T\xde\t\x82\xba,\xe4!\x13\x0br\x15\x19\xa7\xa7\x9a\x022\xd9\xb4\x99\x04=\xfb5\xce\x97R;9\xa6\r\x1b"\xca.\x972bpKu\xb3\xd6\x1e\x05\xd4\x96\xe9\x02\xddl9\xf6N\xf1T#$\x9fo<YB\x17{caIn\x0bf\xa7\x13\xc9$\x06\x97W2{\xbb\xad\x91n{\xf46\x97\x98\xf4}\x82\xe2\x1a\x14V\xa8\xd7\x91L\n\xe25m\xec\x81J\x9b\xa1\x1fm\xf6\xb2\x00'

assert decrypted_payload == expected_decrypted_payload
print()
if sender.encode() in decrypted_payload:
    print('Sender address {0} found in the decrypted payload.'.format(sender))
    print('AGE payload correctly decrypted.')
else:
    print('Sender address {0} NOT found in the decrypted payload.'.format(sender))
    print('Cannot confirm AGE payload was correctly decrypted.')

validage.py

import hashlib

from myage import age_decrypt

from bip_utils import Bech32Decoder

from cryptography.hazmat.primitives import serialization
from cryptography.hazmat.primitives.asymmetric.x25519 import X25519PrivateKey

# official AGE test vector
# https://github.com/C2SP/CCTV/blob/3ec4d716e80597545ed285cf62af3dded3a14f65/age/testdata/x25519

expected_payload_hash = '013f54400c82da08037759ada907a8b864e97de81c088a182062c4b5622fd2ab'
file_key = '59454c4c4f57205355424d4152494e45'
identity = 'AGE-SECRET-KEY-1XMWWC06LY3EE5RYTXM9MFLAZ2U56JJJ36S0MYPDRWSVLUL66MV4QX3S7F6'

age_payload = 'age-encryption.org/v1\n-> X25519 TEiF0ypqr+bpvcqXNyCVJpL7OuwPdVwPL7KQEbFDOCc\nEmECAEcKN+n/Vs9SbWiV+Hu0r+E8R77DdWYyd83nw7U\n--- Vn+54jqiiUCE+WZcEVY3f1sqHjlu/z1LCQ/T7Xm7qI0\n'.encode('utf-8') + bytes.fromhex('eecf62c7ce91b433274e68d4f2f9134cb74c5bfef7beaa52c8f0bc0e992c1e8331fb66')

ephemeral_share = b'TEiF0ypqr+bpvcqXNyCVJpL7OuwPdVwPL7KQEbFDOCc'
encrypted_file_key = b'EmECAEcKN+n/Vs9SbWiV+Hu0r+E8R77DdWYyd83nw7U'
header_mac = b'Vn+54jqiiUCE+WZcEVY3f1sqHjlu/z1LCQ/T7Xm7qI0'

mac_message = b'age-encryption.org/v1\n-> X25519 TEiF0ypqr+bpvcqXNyCVJpL7OuwPdVwPL7KQEbFDOCc\nEmECAEcKN+n/Vs9SbWiV+Hu0r+E8R77DdWYyd83nw7U\n---'
reconstructed_mac_message = b'age-encryption.org/v1\n-> X25519 ' + ephemeral_share + b'\n' + encrypted_file_key + b'\n---'
assert reconstructed_mac_message == mac_message

encrypted_payload = bytes.fromhex(
    'eecf62c7ce91b433274e68d4f2f9134cb74c5bfef7beaa52c8f0bc0e992c1e8331fb66')

x25519sk = Bech32Decoder.Decode('age-secret-key-', identity.lower())
x25519pk = X25519PrivateKey.from_private_bytes(x25519sk).public_key().public_bytes(
    encoding=serialization.Encoding.Raw, format=serialization.PublicFormat.Raw)

assert x25519sk.hex() == '36dcec3f5f24739a0c8b36cbb4ffa25729a94a51d41fb205a37419fe7f5adb2a'
assert x25519pk.hex() == '74564b66a8cb154674c683308072b53d469d6641df590a783a20b5fb91504c29'

decrypted_payload = age_decrypt(
    mac_message,
    ephemeral_share,
    encrypted_file_key,
    header_mac,
    encrypted_payload,
    x25519pk,
    x25519sk,
    ignore_hmac=False,
    expected_decrypted_file_key=bytes.fromhex(file_key))

print()
print('decrypted payload is')
print(decrypted_payload)

decrypted_payload_hashed = hashlib.sha256(decrypted_payload).digest()
assert decrypted_payload_hashed.hex() == expected_payload_hash

Hey @marekyggdrasil, so I ran slateage.py on Windows, macOS, and Linux, but I wasn't able to recreate the issue you're experiencing with hmac.verify failing when ignore_hmac is False.

Here's the values for relevant variables between lines 65-80 in myage.py that successfully verify the HMAC to assist you with troubleshooting this. These values also match the ones generated by the Node.js age example that I made awhile ago when using it with the slatepack and private key from slateage.py.

hkdfval = HKDF(
    algorithm=hashes.SHA256(),
    length=32,
    salt=salt,
    info=HEADER_HKDF_LABEL,
    backend=default_backend()
).derive(decrypted_file_key)

print(binascii.hexlify(hkdfval)) #e9e4ff52ce1c1c2f7c8d280fc6f869ade9bc005b28c07a36c17ee8839e59527f

hmac = HMAC(
    key=hkdfval,
    algorithm=hashes.SHA256(),
    backend=default_backend()
)

print(mac_message); #age-encryption.org/v1\n-> X25519 F/H3hf63Hrq2MH+O1naWm6+M8184u3S7oeHQ4VHiMzY\npbUcLLHdT5PGn/oYeiD+tsUO1oexurSZmSVu8Hcv7mc\n-> Es[-grease )vK1eC lVxuS`,6\nSkmgyCDG0wPb0wvGeYp6XbdSyMONfXYoRPLyQSlkse8tDxkTHKwjoz6Y3t2IIqHv\nU/dxyW2iZyZ1UVFRrlNAVbXbEBwqhpaFjDdRcivhbDo1uTL5CDNvJz89w+ivaoUc\n\n---

hmac.update(mac_message)

print(binascii.hexlify(decoded_header_mac)); #9ead2126ffdb57790b46ba8a53dc2591ec45d924d90e605fd2195e7acfc7c96c

if not ignore_hmac:
    hmac.verify(decoded_header_mac)

Good luck!

Hey @NicolasFlamel1 this is a funny thing, I actually did find the reason for it yesterday, but it was late and I forgot about it. The output in the repo also doesn't fail.

So reason was, the ツ core wallet header might contain more than one newline character

age-encryption.org/v1\n-> X25519 F/H3hf63Hrq2MH+O1naWm6+M8184u3S7oeHQ4VHiMzY\npbUcLLHdT5PGn/oYeiD+tsUO1oexurSZmSVu8Hcv7mc\n-> Es[-grease )vK1eC lVxuS`,6\nSkmgyCDG0wPb0wvGeYp6XbdSyMONfXYoRPLyQSlkse8tDxkTHKwjoz6Y3t2IIqHv\nU/dxyW2iZyZ1UVFRrlNAVbXbEBwqhpaFjDdRcivhbDo1uTL5CDNvJz89w+ivaoUc\n\n---

( note the two \n\n before --- ).

Most libraries would deserialize the header and then serialize it once again to produce MAC message for verification. The serialized message will lose this extra \n before ---. This was same with pyage I used and that also the case with grin++.

If we find a way to put extra newline character in re-serialized AGE headers we could re-enable the HMAC verification in grin++.