Aut64 cryptographic functions in C++

aut64.h

// Aut64 cryptographic module, Gabriel Valky 2020
// based on https://github.com/hkscy/AUT64
// rewritten from python to C++ and implemented decryption function

#include <iostream>
#include <iomanip>

#define COUNT(a) (sizeof(a)/sizeof(a[0]))

class Aut64
{
public:
    typedef int pbox_t[8];
    typedef int key_reg_t[8];
    typedef int sbox_t[16];
    typedef int message_t[8];

private:
    constexpr static const int table_ln[] = {
        0x4, 0x5, 0x6, 0x7, 0x0, 0x1, 0x2, 0x3, // Round 0
        0x5, 0x4, 0x7, 0x6, 0x1, 0x0, 0x3, 0x2, // Round 1
        0x6, 0x7, 0x4, 0x5, 0x2, 0x3, 0x0, 0x1, // ...
        0x7, 0x6, 0x5, 0x4, 0x3, 0x2, 0x1, 0x0,
        0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7,
        0x1, 0x0, 0x3, 0x2, 0x5, 0x4, 0x7, 0x6,
        0x2, 0x3, 0x0, 0x1, 0x6, 0x7, 0x4, 0x5,
        0x3, 0x2, 0x1, 0x0, 0x7, 0x6, 0x5, 0x4,
        0x5, 0x4, 0x7, 0x6, 0x1, 0x0, 0x3, 0x2,
        0x4, 0x5, 0x6, 0x7, 0x0, 0x1, 0x2, 0x3,
        0x7, 0x6, 0x5, 0x4, 0x3, 0x2, 0x1, 0x0, // ...
        0x6, 0x7, 0x4, 0x5, 0x2, 0x3, 0x0, 0x1}; // Round 11
                
    constexpr static const int table_un[] = {
        0x1, 0x0, 0x3, 0x2, 0x5, 0x4, 0x7, 0x6, // Round 0
        0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, // Round 1
        0x3, 0x2, 0x1, 0x0, 0x7, 0x6, 0x5, 0x4, // ...
        0x2, 0x3, 0x0, 0x1, 0x6, 0x7, 0x4, 0x5,
        0x5, 0x4, 0x7, 0x6, 0x1, 0x0, 0x3, 0x2,
        0x4, 0x5, 0x6, 0x7, 0x0, 0x1, 0x2, 0x3,
        0x7, 0x6, 0x5, 0x4, 0x3, 0x2, 0x1, 0x0,
        0x6, 0x7, 0x4, 0x5, 0x2, 0x3, 0x0, 0x1,
        0x3, 0x2, 0x1, 0x0, 0x7, 0x6, 0x5, 0x4,
        0x2, 0x3, 0x0, 0x1, 0x6, 0x7, 0x4, 0x5,
        0x1, 0x0, 0x3, 0x2, 0x5, 0x4, 0x7, 0x6, // ...
        0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7}; // Round 11

    // AUT64 Compression function substituiton table T_offset
    // Input Nibbble:   0    1    2    3    4    5    6    7    8    9    A    B    C    D    E    F  | Key Nibble
    constexpr static const int table_offset[] = {
        0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, // 0
        0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0xA, 0xB, 0xC, 0xD, 0xE, 0xF, // 1
        0x0, 0x2, 0x4, 0x6, 0x8, 0xA, 0xC, 0xE, 0x3, 0x1, 0x7, 0x5, 0xB, 0x9, 0xF, 0xD, // 2
        0x0, 0x3, 0x6, 0x5, 0xC, 0xF, 0xA, 0x9, 0xB, 0x8, 0xD, 0xE, 0x7, 0x4, 0x1, 0x2, // 3
        0x0, 0x4, 0x8, 0xC, 0x3, 0x7, 0xB, 0xF, 0x6, 0x2, 0xE, 0xA, 0x5, 0x1, 0xD, 0x9, // 4
        0x0, 0x5, 0xA, 0xF, 0x7, 0x2, 0xD, 0x8, 0xE, 0xB, 0x4, 0x1, 0x9, 0xC, 0x3, 0x6, // 5
        0x0, 0x6, 0xC, 0xA, 0xB, 0xD, 0x7, 0x1, 0x5, 0x3, 0x9, 0xF, 0xE, 0x8, 0x2, 0x4, // 6
        0x0, 0x7, 0xE, 0x9, 0xF, 0x8, 0x1, 0x6, 0xD, 0xA, 0x3, 0x4, 0x2, 0x5, 0xC, 0xB, // 7
        0x0, 0x8, 0x3, 0xB, 0x6, 0xE, 0x5, 0xD, 0xC, 0x4, 0xF, 0x7, 0xA, 0x2, 0x9, 0x1, // 8
        0x0, 0x9, 0x1, 0x8, 0x2, 0xB, 0x3, 0xA, 0x4, 0xD, 0x5, 0xC, 0x6, 0xF, 0x7, 0xE, // 9
        0x0, 0xA, 0x7, 0xD, 0xE, 0x4, 0x9, 0x3, 0xF, 0x5, 0x8, 0x2, 0x1, 0xB, 0x6, 0xC, // A
        0x0, 0xB, 0x5, 0xE, 0xA, 0x1, 0xF, 0x4, 0x7, 0xC, 0x2, 0x9, 0xD, 0x6, 0x8, 0x3, // B
        0x0, 0xC, 0xB, 0x7, 0x5, 0x9, 0xE, 0x2, 0xA, 0x6, 0x1, 0xD, 0xF, 0x3, 0x4, 0x8, // C
        0x0, 0xD, 0x9, 0x4, 0x1, 0xC, 0x8, 0x5, 0x2, 0xF, 0xB, 0x6, 0x3, 0xE, 0xA, 0x7, // D
        0x0, 0xE, 0xF, 0x1, 0xD, 0x3, 0x2, 0xC, 0x9, 0x7, 0x6, 0x8, 0x4, 0xA, 0xB, 0x5, // E
        0x0, 0xF, 0xD, 0x2, 0x9, 0x6, 0x4, 0xB, 0x1, 0xE, 0xC, 0x3, 0x8, 0x7, 0x5, 0xA}; // F
    
    constexpr static const int table_sub[] = {
        0x0, 0x1, 0x9, 0xE, 0xD, 0xB, 0x7, 0x6, 0xF, 0x2, 0xC, 0x5, 0xA, 0x4, 0x3, 0x8};
    
    void clear(message_t& m)
    {
        memset(m, 0, sizeof(m));
    }
    
    void copy(message_t& a, const message_t& b)
    {
        memcpy(a, b, sizeof(message_t));
    }
    
    template <typename T> void inverse(T& out, const T& in)
    {
        for (int i=0; i<COUNT(out); i++)
        {
            assert(in[i] < COUNT(out));
            out[in[i]] = i;
        }
    }
    
    void apply_pbox(message_t& m, const pbox_t& pbox, int n = 1)
    {
        while (n--)
        {
            message_t n;
            clear(n);
            for (int i=0; i<COUNT(m); i++)
                n[pbox[i]] = m[i];
            copy(m, n);
        }
    }
    
    int apply_pbox_bitwise(int input_data, const pbox_t& pbox)
    {
        int result = 0;

        for (int bit=0; bit<8; bit++)
            if (input_data & (1 << bit))
                result |= 1 << pbox[bit];

        return result;
    }

    int apply_sbox(int byte, const sbox_t& sbox)
    {
        return sbox[byte & 0xf] | (sbox[(byte >> 4) & 0x0f] << 4);
    }

    int compress_sum_low(const message_t& state, const key_reg_t& key_reg, int roundN)
    {
        int r6 = 0;
        for (int byte=0; byte<7; byte++)
        {
            int ln = state[byte] & 0xf; // get lower byte nibble
            int lk = key_reg[table_ln[8*roundN + byte]]; // get key nibble
            int p0 = ln | (lk << 4); // combine state and key nibble
            r6 ^= table_offset[p0];
        }
        return r6;
    }

    int compress_sum_high(const message_t& state, const key_reg_t& key_reg, int roundN)
    {
        int r5 = 0;
        for (int byte=0; byte<7; byte++)
        {
            int un = state[byte] >> 4; // get upper byte nibble
            int uk = key_reg[table_un[8*roundN + byte]]; //get key nibble
            int p1 = un | (uk << 4); // combine state and key nibble uk,un
            r5 ^= table_offset[p1];
        }
        return r5;
    }

    int compress_substitute(int ln, int lk)
    {
        int ls = (table_sub[lk] << 4) & 0xf0;   // Substitute and move lower into upper nibble

        // Find index of ln in table_offset at ls+(0,1,2,...,15)
        int i_ln;
        for (i_ln=0; i_ln<16; i_ln++)
            if (table_offset[ls + i_ln] == ln)
                return i_ln;
        
        assert(0);
        return 0;
    }
    
    int compress_unsubstitute(int ln, int lk)
    {
        if (lk == 0)
            assert(0);
        
        int ls = (table_sub[lk] << 4) & 0xf0;   // Substitute and move lower into upper nibble
        return table_offset[ls + ln];
    }
    
    int compress(const message_t& state, const key_reg_t& key_reg, int roundN)
    {
        // upper nibble in byte ##
        int r5 = compress_sum_high(state, key_reg, roundN);
        r5 ^= compress_substitute(state[7] >> 4, key_reg[table_un[8*roundN + 7]]);

        // lower nibble in byte ##
        int r6 = compress_sum_low(state, key_reg, roundN);
        r6 ^= compress_substitute(state[7] & 0xf, key_reg[table_ln[8*roundN + 7]]);
            
        return ((r5 << 4) & 0xf0) | (r6 & 0x0f); //r5,r6
    }
        
    int decompress(const message_t& state, const key_reg_t& key_reg, int roundN)
    {
        int high = state[7] >> 4;
        int r5 = high ^ compress_sum_high(state, key_reg, roundN);
        high = compress_unsubstitute(r5, key_reg[table_un[8*roundN + 7]]);

        int low = state[7] & 0xf;
        int r6 = low ^ compress_sum_low(state, key_reg, roundN);
        low = compress_unsubstitute(r6, key_reg[table_ln[8*roundN + 7]]);

        return ((high << 4) & 0xf0) | (low & 0x0f);
    }

public:
    void dump(const message_t& m)
    {
        std::cout << "message: ";
        for (int i=0; i<COUNT(m); i++)
        {
            std::cout << std::hex << std::setw(2) << std::setfill('0');
            std::cout << m[i] << " ";
        }
        std::cout << "\n";
    }

    void encrypt(message_t& m, const key_reg_t& key_reg, const pbox_t& pbox, const sbox_t& sbox, int nRounds)
    {
        for (int round=0; round<nRounds; round++)
        {
            apply_pbox(m, pbox);
            int step1 = compress(m, key_reg, round);
            int step2 = apply_sbox(step1, sbox);
            int step3 = apply_pbox_bitwise(step2, pbox);
            int step4 = apply_sbox(step3, sbox);
            m[7] = step4;
        }
    }
    
    void decrypt(message_t& m, const key_reg_t& key_reg, const pbox_t& pbox, const sbox_t& sbox, int nRounds)
    {
        sbox_t sbox_inverse;
        inverse<sbox_t>(sbox_inverse, sbox);

        pbox_t pbox_inverse;
        inverse<pbox_t>(pbox_inverse, pbox);

        for (int round=0; round<nRounds; round++)
        {
            int step4 = m[7];
            int step3 = apply_sbox(step4, sbox_inverse);
            int step2 = apply_pbox_bitwise(step3, pbox_inverse);
            m[7] = apply_sbox(step2, sbox_inverse);
            m[7] = decompress(m, key_reg, nRounds-1-round);
            apply_pbox(m, pbox_inverse);
        }
    }
};

int main(int argc, const char * argv[])
{
    Aut64::pbox_t pbox = {2, 0, 6, 5, 7, 4, 3, 1};
    Aut64::key_reg_t key_reg = {10, 8, 4, 14, 5, 4, 8, 11};
    Aut64::sbox_t sbox = {5, 11, 7, 12, 4, 8, 0, 3, 13, 9, 6, 1, 2, 14, 10, 15};
    Aut64::message_t message = {1, 2, 3, 4, 5, 6, 7, 8};
    int nRounds = 8;

    Aut64 aut64;
    aut64.dump(message);
    aut64.encrypt(message, key_reg, pbox, sbox, nRounds);
    aut64.dump(message);
    aut64.decrypt(message, key_reg, pbox, sbox, nRounds);
    aut64.dump(message);

    // message: 01 02 03 04 05 06 07 08
    // message: ab 21 b6 c5 22 4f 6d 9d
    // message: 01 02 03 04 05 06 07 08
    return 0;
}