Obfuscation with byte substitution

mask.cpp

//
// Simple obfuscation using byte substitution
//

#include <cstdio>
#include <cstdint>
#include <cstdlib>
#include <cstring>
#include <cmath>
#include <ctime>

void
dump(const char *str, void *buf, size_t len) {
    uint8_t *x = (uint8_t*)buf;
    
    printf("\n\n%s:\n", str);
    
    for (size_t i=0; i<len; i++) {
        if (!(i & 15)) putchar('\n');
        printf(" %02X", x[i]);
    }
}

typedef struct _mask_ctx {
    uint8_t sbox[256];
    uint8_t key[16];
    uint8_t sbox_inv[256];
} mask_ctx;

extern "C" {
    void init_mask_x86(mask_ctx *c);
    void mask_x86(void *sbox, size_t len, void *buf);
}

// initialise using RC4
void
init_mask(mask_ctx *c) {
    // initialise sbox
    for (size_t i=0; i<256; i++) {
        c->sbox[i] = (uint8_t)i;
    }
    // shuffle sbox
    for (size_t i=0, j=0; i<256; i++) {
        j = (j + (c->sbox[i] + c->key[i % 16])) & 255;
        uint8_t t = c->sbox[i] & 255;
        c->sbox[i] = c->sbox[j];
        c->sbox[j] = t;
    }
    // create inverse
    for (size_t i=0; i<256; i++) {
        c->sbox_inv[c->sbox[i]] = i;
    }
}

// mask or unmask
void
mask(uint8_t *sbox, size_t len, void *buf) {
    uint8_t *in = (uint8_t*)buf;
    uint8_t *out = (uint8_t*)buf;
    
    for (size_t i=0; i<len; i++) {
        out[i] = sbox[in[i]];
    }
}

int
main(void) {
    uint8_t inbuf[32];
    mask_ctx c={};
    
    // initialise key
    srand(time(0));
    for(int i=0; i<16; i++) c.key[i]=rand();
    
    // initialise sboxes
    init_mask(&c);
    
    dump("sbox", c.sbox, 256);
    dump("inverse sbox", c.sbox_inv, 256);
    
    // initialise test input
    for(int i=0; i<sizeof(inbuf); i++) inbuf[i]=i;
    dump("raw", inbuf, sizeof(inbuf));
    
    // mask inbuf
    mask(c.sbox_inv, sizeof(inbuf), inbuf);
    dump("encoded", inbuf, sizeof(inbuf));
    
    // unmask inbuf
    mask(c.sbox, sizeof(inbuf), inbuf);
    dump("decoded", inbuf, sizeof(inbuf));
    
    return 0;
}


/**


;
; Simple obfuscation using byte substitution.
;
    
    bits 32
    
%ifndef BIN
    global _init_mask_x86
    global init_mask_x86
    
    global _mask_x86
    global mask_x86
%endif

    section .text
    
    ;
    ; void init_mask_x86(mask_ctx*c);
    ;
_init_mask_x86:
init_mask_x86:
    pushad
    mov    edi, [esp+32+4]
    push   edi
    pop    esi
    xor    ecx, ecx          ; i=0
    mul    ecx
initialise_sbox:
    stosb                    ; c->sbox[i]=i
    inc    al                ; i++
    jnz    initialise_sbox   ; i<256
shuffle_sbox:
    ; j = (j + (c->sbox[i] + c->key[i % 16])) & 255;
    mov    al, dl            ; t = i % 16
    and    al, 15            ;
    
    add    cl, [edi+eax]     ; j += c->key[i % 16]
    mov    al, [esi+edx]     ; t = c->sbox[i]
    add    cl, al            ; j += c->sbox[i]
    xchg   al, [esi+ecx]     ; swap(t, s[j])
    mov    [esi+edx], al
    
    inc    dl                ; i++
    jnz    shuffle_sbox      ; i<256
    add    edi, 16
create_inverse:
    lodsb                    ; sbox_inv[sbox[i]] = i
    mov    [edi+eax], dl     ; 
    inc    dl
    jnz    create_inverse
    
    popad
    ret
    
    ;
    ; void mask_x86(void *sbox, size_t inlen, void *inbuf);
    ;
mask_x86:
_mask_x86:
    pushad
    lea    esi, [esp+32+4]
    lodsd
    xchg   ebx, eax          ; bx = sbox
    lodsd
    xchg   ecx, eax          ; cx = inlen
    lodsd
    xchg   esi, eax          ; si = inbuf
    push   esi
    pop    edi
mask_loop:
    lodsb                    ; al = in[i]
    xlatb                    ; al = sbox[al]
    stosb                    ; out[i] = al
    loop   mask_loop
    popad
    ret
    
*/