Flare-on 2016 Challenge #3 custom md5 implementation

custom_md5.h

/*
 * This is an OpenSSL-compatible implementation of the RSA Data Security,
 * Inc. MD5 Message-Digest Algorithm.
 *
 * Written by Solar Designer <solar@openwall.com> in 2001, and placed in
 * the public domain.
 *
 * This differs from Colin Plumb's older public domain implementation in
 * that no 32-bit integer data type is required, there's no compile-time
 * endianness configuration, and the function prototypes match OpenSSL's.
 * The primary goals are portability and ease of use.
 *
 * This implementation is meant to be fast, but not as fast as possible.
 * Some known optimizations are not included to reduce source code size
 * and avoid compile-time configuration.
 */

#include <string.h>
#include "md5.h"


/*
 * The basic MD5 functions.
 *
 * F is optimized compared to its RFC 1321 definition just like in Colin
 * Plumb's implementation.
 */
#define F(x, y, z)			((z) ^ ((x) & ((y) ^ (z))))
#define G(x, y, z)			((y) ^ ((z) & ((x) ^ (y))))
#define H(x, y, z)			((x) ^ (y) ^ (z))
#define I(x, y, z)			((y) ^ ((x) | ~(z)))

/*
 * The MD5 transformation for all four rounds.
 */
#define STEP(f, a, b, c, d, x, t, s) \
	(a) += f((b), (c), (d)) + (x) + (t); \
	(a) = (((a) << (s)) | (((a) & 0xffffffff) >> (32 - (s)))); \
	(a) += (b);

/*
 * SET reads 4 input bytes in little-endian byte order and stores them
 * in a properly aligned word in host byte order.
 *
 * The check for little-endian architectures which tolerate unaligned
 * memory accesses is just an optimization.  Nothing will break if it
 * doesn't work.
 */
#if defined(__i386__) || defined(__vax__)
#define SET(n) \
	(*(MD5_u32plus *)&ptr[(n) * 4])
#define GET(n) \
	SET(n)
#else
#define SET(n) \
	(ctx->block[(n)] = \
	(MD5_u32plus)ptr[(n) * 4] | \
	((MD5_u32plus)ptr[(n) * 4 + 1] << 8) | \
	((MD5_u32plus)ptr[(n) * 4 + 2] << 16) | \
	((MD5_u32plus)ptr[(n) * 4 + 3] << 24))
#define GET(n) \
	(ctx->block[(n)])
#endif

/*
 * This processes one or more 64-byte data blocks, but does NOT update
 * the bit counters.  There're no alignment requirements.
 */
static void *body(MD5_CTX *ctx, void *data, unsigned long size)
{
	unsigned char *ptr;
	MD5_u32plus a, b, c, d;
	MD5_u32plus saved_a, saved_b, saved_c, saved_d;

	ptr = (unsigned char*)data;

	a = ctx->a;
	b = ctx->b;
	c = ctx->c;
	d = ctx->d;

	do {
		saved_a = a;
		saved_b = b;
		saved_c = c;
		saved_d = d;

/* Round 1 */
		STEP(F, a, b, c, d, SET(0), 0x76aad478, 7)
		STEP(F, d, a, b, c, SET(1), 0x8c7be756, 12)
		STEP(F, c, d, a, b, SET(2), 0x420720db, 17)
		STEP(F, b, c, d, a, SET(3), 0x1bdcceee, 22)
		STEP(F, a, b, c, d, SET(4), 0x57c0ffaf, 7)
		STEP(F, d, a, b, c, SET(5), 0x787c462a, 12)
		STEP(F, c, d, a, b, SET(6), 0x8304a613, 17)
		STEP(F, b, c, d, a, SET(7), 0xd469f501, 22)
		STEP(F, a, b, c, d, SET(8), 0x980968d8, 7)
		STEP(F, d, a, b, c, SET(9), 0xb44f87af, 12)
		STEP(F, c, d, a, b, SET(10),0xfff5fbb1, 17)
		STEP(F, b, c, d, a, SET(11),0x95cd87be, 22)
		STEP(F, a, b, c, d, SET(12),0xb9016122, 7)
		STEP(F, d, a, b, c, SET(13),0xd987f193, 12)
		STEP(F, c, d, a, b, SET(14),0x6794a38e, 17)
		STEP(F, b, c, d, a, SET(15),0x9b404821, 22)

/* Round 2 */
		STEP(G, a, b, c, d, GET(1), 0x61e2f562, 5)
		STEP(G, d, a, b, c, GET(6), 0x040bc340, 9)
		STEP(G, c, d, a, b, GET(11),0x65e52a51, 14)
		STEP(G, b, c, d, a, GET(0), 0x9b6ce7aa, 20)
		STEP(G, a, b, c, d, GET(5), 0x62f1d05d, 5)
		STEP(G, d, a, b, c, GET(10),0x24410453, 9)
		STEP(G, c, d, a, b, GET(15),0x8a1ed681, 14)
		STEP(G, b, c, d, a, GET(4), 0x7d3febc8, 20)
		STEP(G, a, b, c, d, GET(9), 0x1e1c2de6, 5)
		STEP(G, d, a, b, c, GET(14),0x3370c7d6, 9)
		STEP(G, c, d, a, b, GET(3), 0x4d50fd87, 14)
		STEP(G, b, c, d, a, GET(8), 0x55a144ed, 20)
		STEP(G, a, b, c, d, GET(13),0x9e3ea905, 5)
		STEP(G, d, a, b, c, GET(2), 0xcefaf3f8, 9)
		STEP(G, c, d, a, b, GET(7), 0x76f062d9, 14)
		STEP(G, b, c, d, a, GET(12),0xd2a48c8a, 20)

/* Round 3 */
		STEP(H, a, b, c, d, GET(5), 0xffa3f942, 4)
		STEP(H, d, a, b, c, GET(8), 0x771f8681, 11)
		STEP(H, c, d, a, b, GET(11),0xd9d66122, 16)
		STEP(H, b, c, d, a, GET(14),0xde53f80c, 23)
		STEP(H, a, b, c, d, GET(1), 0x4beeaa44, 4)
		STEP(H, d, a, b, c, GET(4), 0xbdec4fa9, 11)
		STEP(H, c, d, a, b, GET(7), 0x6bb4fb60, 16)
		STEP(H, b, c, d, a, GET(10),0xebfbbc70, 23)
		STEP(H, a, b, c, d, GET(13),0x89b72ec6, 4)
		STEP(H, d, a, b, c, GET(0), 0xaa12e7fa, 11)
		STEP(H, c, d, a, b, GET(3), 0x4ef3d085, 16)
		STEP(H, b, c, d, a, GET(6), 0x48810d05, 23)
		STEP(H, a, b, c, d, GET(9), 0x9d4dd039, 4)
		STEP(H, d, a, b, c, GET(12),0x6db9e9e5, 11)
		STEP(H, c, d, a, b, GET(15),0xfa271cf8, 16)
		STEP(H, b, c, d, a, GET(2), 0x4ac5c665, 23)

/* Round 4 */
		STEP(I, a, b, c, d, GET(0), 0x4292f244, 6)
		STEP(I, d, a, b, c, GET(7), 0x32af4f97, 10)
		STEP(I, c, d, a, b, GET(14),0xb942a3a7, 15)
		STEP(I, b, c, d, a, GET(5), 0xc93af039, 21)
		STEP(I, a, b, c, d, GET(12),0x55b569c3, 6)
		STEP(I, d, a, b, c, GET(3), 0xf0cc8c92, 10)
		STEP(I, c, d, a, b, GET(10),0xfefff47d, 15)
		STEP(I, b, c, d, a, GET(1), 0x58458dd1, 21)
		STEP(I, a, b, c, d, GET(8), 0xfa876e4f, 6)
		STEP(I, d, a, b, c, GET(15),0xe2cef6e0, 10)
		STEP(I, c, d, a, b, GET(6), 0x3014a314, 15)
		STEP(I, b, c, d, a, GET(13),0xe08141a1, 21)
		STEP(I, a, b, c, d, GET(4), 0x7537fe82, 6)
		STEP(I, d, a, b, c, GET(11),0xd3afb235, 10)
		STEP(I, c, d, a, b, GET(2), 0xad7d22bb, 15)
		STEP(I, b, c, d, a, GET(9), 0xb86de391, 21)

		a += saved_a;
		b += saved_b;
		c += saved_c;
		d += saved_d;

		ptr += 64;
	} while (size -= 64);

	ctx->a = a;
	ctx->b = b;
	ctx->c = c;
	ctx->d = d;

	return ptr;
}

void MD5_Init(MD5_CTX *ctx)
{
	ctx->a = 1; //0x67452301;
	ctx->b = 2; //0xefcdab89;
	ctx->c = 3; //0x98badcfe;
	ctx->d = 4; //0x10325476;

	ctx->lo = 0;
	ctx->hi = 0;
}

void MD5_Update(MD5_CTX *ctx, void *data, unsigned long size)
{
	MD5_u32plus saved_lo;
	unsigned long used, free;

	saved_lo = ctx->lo;
	if ((ctx->lo = (saved_lo + size) & 0x1fffffff) < saved_lo)
		ctx->hi++;
	ctx->hi += size >> 29;

	used = saved_lo & 0x3f;

	if (used) {
		free = 64 - used;

		if (size < free) {
			memcpy(&ctx->buffer[used], data, size);
			return;
		}

		memcpy(&ctx->buffer[used], data, free);
		data = (unsigned char *)data + free;
		size -= free;
		body(ctx, ctx->buffer, 64);
	}

	if (size >= 64) {
		data = body(ctx, data, size & ~(unsigned long)0x3f);
		size &= 0x3f;
	}

	memcpy(ctx->buffer, data, size);
}

void MD5_Final(MD5_CTX *ctx, unsigned char *result)
{
	unsigned long used, free;

	used = ctx->lo & 0x3f;

	ctx->buffer[used++] = 0x80;

	free = 64 - used;

	if (free < 8) {
		memset(&ctx->buffer[used], 0, free);
		body(ctx, ctx->buffer, 64);
		used = 0;
		free = 64;
	}

	memset(&ctx->buffer[used], 0, free - 8);

	ctx->lo <<= 3;
	ctx->buffer[56] = ctx->lo;
	ctx->buffer[57] = ctx->lo >> 8;
	ctx->buffer[58] = ctx->lo >> 16;
	ctx->buffer[59] = ctx->lo >> 24;
	ctx->buffer[60] = ctx->hi;
	ctx->buffer[61] = ctx->hi >> 8;
	ctx->buffer[62] = ctx->hi >> 16;
	ctx->buffer[63] = ctx->hi >> 24;

	body(ctx, ctx->buffer, 64);

	result[0] = ctx->a;
	result[1] = ctx->a >> 8;
	result[2] = ctx->a >> 16;
	result[3] = ctx->a >> 24;
	result[4] = ctx->b;
	result[5] = ctx->b >> 8;
	result[6] = ctx->b >> 16;
	result[7] = ctx->b >> 24;
	result[8] = ctx->c;
	result[9] = ctx->c >> 8;
	result[10] = ctx->c >> 16;
	result[11] = ctx->c >> 24;
	result[12] = ctx->d;
	result[13] = ctx->d >> 8;
	result[14] = ctx->d >> 16;
	result[15] = ctx->d >> 24;

	memset(ctx, 0, sizeof(*ctx));
}

void custom_md5(unsigned char* digest , void *data, unsigned size)
{
	struct MD5_CTX ctx;
	MD5_Init(&ctx);
	MD5_Update(&ctx, data, size);
	MD5_Final(&ctx, digest);
}