dead simple crypto obfuscator with gcc (see https://brokenpi.pe)

plugin_noxor.c

/*
 * The MIT License (MIT)
 * Copyright (c) <2016> <Nahim El Atmani>
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 * noxor_plugin.c
 *
 * This plugin contains an obfuscating pass that affects all functions tagget as
 * __attribute__((noxor)). It replaces the xor intruction with logical
 * equivalent (like four NAND, 5 NOR or 2 NAND and an OR gate).
 *
 * A statement of the form x = var1 ^ var2 will be transformed into a logical
 * equivalent
 *
 */

#include <gcc-plugin.h>
#include <tree.h>
#include <tree-pass.h>
#include <gimple.h>
#include <gimple-builder.h>
#include <gimple-iterator.h>
#include <stdio.h>
#include <vec.h>

#define RAND(MIN, MAX) ((int)(((double)rand() / ((double)RAND_MAX + 1.0)) * ((MAX) - (MIN) + 1) + (MIN)))

int plugin_is_GPL_compatible;
vec<tree> fnoxor = vNULL;
bool nofilter;
#ifdef DEBUG
int xor_removed;
#endif

enum xor_flavour {
	mixed,
	nand,
	nor,
	_nb_flavour
};

void (*noxor[_nb_flavour])(gimple_stmt_iterator *, tree *);

static tree handle_noxor_attribute(tree *node, tree name, tree args, int flags, 
				   bool *no_add_attrs)
{
	fnoxor.safe_push(*node);

	return NULL_TREE;
}

struct plugin_info noxor_info = {
	.version = "5.3",
	.help = "plugin_noxor add a GCC pass to replace every xor in "
		"function decorated by __attribute__((noxor)) "
		"by logical equivalents. The main purpose of this "
		"pluggin is to have less visible crypto logics in "
		"the generated assembly when reverse engineered."
};

static struct attribute_spec noxor_attr = { 
	.name = "noxor",
	.min_length = 0, 
	.max_length = 0, 
	.decl_required = true,
	.type_required = false,
	.function_type_required = false,
	.handler = handle_noxor_attribute,
	.affects_type_identity = false
};

static bool is_noxorfunc(tree decl)
{
	unsigned i;
	tree fn;

	FOR_EACH_VEC_ELT(fnoxor, i, fn)
		if (fn == decl)
			return true;
	return false;
}

static void register_attributes(void *event_data, void *data)
{
	register_attribute(&noxor_attr);
}

static void noxor_mixed(gimple_stmt_iterator *gsi, tree *ops)
{
	gimple_stmt_iterator old;
	gimple* stmt;
	tree temp[9];

	old = *gsi;
	/* T0 = a */
	temp[0] = create_tmp_var(TREE_TYPE(ops[1]), "TMP");
	stmt = gimple_build_assign(temp[0], VAR_DECL, ops[1]);
	gsi_insert_after(gsi, stmt, GSI_NEW_STMT);
	/* T1 = b */
	temp[1] = create_tmp_var(TREE_TYPE(ops[2]), "TMP");
	stmt = gimple_build_assign(temp[1], VAR_DECL, ops[2]);
	gsi_insert_after(gsi, stmt, GSI_NEW_STMT);
	/* T2 = T0 & T1 */
	temp[2] = create_tmp_var(TREE_TYPE(ops[0]), "TMP");
	stmt = gimple_build_assign(temp[2], BIT_AND_EXPR,
				   temp[1], temp[0]);
	gsi_insert_after(gsi, stmt, GSI_NEW_STMT);
	/* T3 = ~T2 */
	temp[3] = create_tmp_var(TREE_TYPE(ops[0]), "TMP");
	stmt = gimple_build_assign(temp[3], BIT_NOT_EXPR,
				   temp[2]);
	gsi_insert_after(gsi, stmt, GSI_NEW_STMT);
	/* T4 = a */
	temp[4] = create_tmp_var(TREE_TYPE(ops[1]), "TMP");
	stmt = gimple_build_assign(temp[4], VAR_DECL, ops[1]);
	gsi_insert_after(gsi, stmt, GSI_NEW_STMT);
	/* T5 = b */
	temp[5] = create_tmp_var(TREE_TYPE(ops[2]), "TMP");
	stmt = gimple_build_assign(temp[5], VAR_DECL, ops[2]);
	gsi_insert_after(gsi, stmt, GSI_NEW_STMT);
	/* T6 = T4 | T5 */
	temp[6] = create_tmp_var(TREE_TYPE(ops[0]), "TMP");
	stmt = gimple_build_assign(temp[6], BIT_IOR_EXPR,
				   temp[4], temp[5]);
	gsi_insert_after(gsi, stmt, GSI_NEW_STMT);
	/* T7 = T3 & T6 */
	temp[7] = create_tmp_var(TREE_TYPE(ops[0]), "TMP");
	stmt = gimple_build_assign(temp[7], BIT_AND_EXPR,
				   temp[3], temp[6]);
	gsi_insert_after(gsi, stmt, GSI_NEW_STMT);
	stmt = gimple_build_assign(ops[0], VAR_DECL, temp[7]);
	gsi_insert_after(gsi, stmt, GSI_NEW_STMT);

	gsi_remove(&old, true);

#ifdef DEBUG
	puts("xor obfuscated with noxor_mixed");
#endif
}

static void noxor_nand(gimple_stmt_iterator *gsi, tree *ops)
{
	gimple_stmt_iterator old;
	gimple* stmt;
	tree temp[12];

	old = *gsi;

	/* T0 = a */
	temp[0] = create_tmp_var(TREE_TYPE(ops[1]), "TMP");
	stmt = gimple_build_assign(temp[0], VAR_DECL, ops[1]);
	gsi_insert_after(gsi, stmt, GSI_NEW_STMT);
	/* T1 = b */
	temp[1] = create_tmp_var(TREE_TYPE(ops[2]), "TMP");
	stmt = gimple_build_assign(temp[1], VAR_DECL, ops[2]);
	gsi_insert_after(gsi, stmt, GSI_NEW_STMT);
	/* T2 = T0 & T1 */
	temp[2] = create_tmp_var(TREE_TYPE(ops[2]), "TMP");
	stmt = gimple_build_assign(temp[2], BIT_AND_EXPR,
				   temp[1], temp[0]);
	gsi_insert_after(gsi, stmt, GSI_NEW_STMT);
	/* T3 = ~T2 */
	temp[3] = create_tmp_var(TREE_TYPE(ops[2]), "TMP");
	stmt = gimple_build_assign(temp[3], BIT_NOT_EXPR,
				   temp[2]);
	gsi_insert_after(gsi, stmt, GSI_NEW_STMT);
	/* T4 = a */
	temp[4] = create_tmp_var(TREE_TYPE(ops[1]), "TMP");
	stmt = gimple_build_assign(temp[4], VAR_DECL, ops[1]);
	gsi_insert_after(gsi, stmt, GSI_NEW_STMT);
	/* T5 = T3 & T4 */
	temp[5] = create_tmp_var(TREE_TYPE(ops[2]), "TMP");
	stmt = gimple_build_assign(temp[5], BIT_AND_EXPR,
				   temp[3], temp[4]);
	gsi_insert_after(gsi, stmt, GSI_NEW_STMT);
	/* T6 = ~T5 */
	temp[6] = create_tmp_var(TREE_TYPE(ops[2]), "TMP");
	stmt = gimple_build_assign(temp[6], BIT_NOT_EXPR,
				   temp[5]);
	gsi_insert_after(gsi, stmt, GSI_NEW_STMT);
	/* T7 = b */
	temp[7] = create_tmp_var(TREE_TYPE(ops[2]), "TMP");
	stmt = gimple_build_assign(temp[7], VAR_DECL, ops[2]);
	gsi_insert_after(gsi, stmt, GSI_NEW_STMT);
	/* T8 = T7 & T3 */
	temp[8] = create_tmp_var(TREE_TYPE(ops[2]), "TMP");
	stmt = gimple_build_assign(temp[8], BIT_AND_EXPR,
				   temp[7], temp[3]);
	gsi_insert_after(gsi, stmt, GSI_NEW_STMT);
	/* T9 = ~T8 */
	temp[9] = create_tmp_var(TREE_TYPE(ops[2]), "TMP");
	stmt = gimple_build_assign(temp[9], BIT_NOT_EXPR,
				   temp[8]);
	gsi_insert_after(gsi, stmt, GSI_NEW_STMT);
	/* T10 = T6 & T9 */
	temp[10] = create_tmp_var(TREE_TYPE(ops[2]), "TMP");
	stmt = gimple_build_assign(temp[10], BIT_AND_EXPR,
				   temp[6], temp[9]);
	gsi_insert_after(gsi, stmt, GSI_NEW_STMT);
	/* T11 = ~T10 */
	temp[11] = create_tmp_var(TREE_TYPE(ops[2]), "TMP");
	stmt = gimple_build_assign(temp[11], BIT_NOT_EXPR,
				   temp[10]);
	gsi_insert_after(gsi, stmt, GSI_NEW_STMT);
	stmt = gimple_build_assign(ops[0], VAR_DECL, temp[11]);
	gsi_insert_after(gsi, stmt, GSI_NEW_STMT);

	gsi_remove(&old, true);

#ifdef DEBUG
	puts("xor obfuscated with noxor_nand");
#endif
}

static void noxor_nor(gimple_stmt_iterator *gsi, tree *ops)
{
	gimple_stmt_iterator old;
	gimple* stmt;
	tree temp[12];

	old = *gsi;

	/* T0 = a */
	temp[0] = create_tmp_var(TREE_TYPE(ops[1]), "T0");
	stmt = gimple_build_assign(temp[0], VAR_DECL, ops[1]);
	gsi_insert_after(gsi, stmt, GSI_NEW_STMT);
	/* T1 = b */
	temp[1] = create_tmp_var(TREE_TYPE(ops[2]), "T1");
	stmt = gimple_build_assign(temp[1], VAR_DECL, ops[2]);
	gsi_insert_after(gsi, stmt, GSI_NEW_STMT);
	/* T10 = a */
	temp[10] = create_tmp_var(TREE_TYPE(ops[1]), "T10");
	stmt = gimple_build_assign(temp[10], VAR_DECL, ops[1]);
	gsi_insert_after(gsi, stmt, GSI_NEW_STMT);
	/* T11 = b */
	temp[11] = create_tmp_var(TREE_TYPE(ops[2]), "T11");
	stmt = gimple_build_assign(temp[11], VAR_DECL, ops[2]);
	gsi_insert_after(gsi, stmt, GSI_NEW_STMT);
	/* T2 = ~(T0 | T0) = ~T0 */
	temp[2] = create_tmp_var(TREE_TYPE(ops[0]), "T2");
	stmt = gimple_build_assign(temp[2], BIT_NOT_EXPR, temp[0]);
	gsi_insert_after(gsi, stmt, GSI_NEW_STMT);
	/* T3 = ~(T1 | T1) = ~T1 */
	temp[3] = create_tmp_var(TREE_TYPE(ops[0]), "T3");
	stmt = gimple_build_assign(temp[3], BIT_NOT_EXPR, temp[1]);
	gsi_insert_after(gsi, stmt, GSI_NEW_STMT);
	/* T4 = T10 | T11 */
	temp[4] = create_tmp_var(TREE_TYPE(ops[0]), "T4");
	stmt = gimple_build_assign(temp[4], BIT_IOR_EXPR, temp[10], temp[11]);
	gsi_insert_after(gsi, stmt, GSI_NEW_STMT);
	/* T5 = ~T4 */
	temp[5] = create_tmp_var(TREE_TYPE(ops[0]), "T5");
	stmt = gimple_build_assign(temp[5], BIT_NOT_EXPR, temp[4]);
	gsi_insert_after(gsi, stmt, GSI_NEW_STMT);
	/* T6 = T2 | T3 */
	temp[6] = create_tmp_var(TREE_TYPE(ops[0]), "T6");
	stmt = gimple_build_assign(temp[6], BIT_IOR_EXPR, temp[2], temp[3]);
	gsi_insert_after(gsi, stmt, GSI_NEW_STMT);
	/* T7 = ~T6 */
	temp[7] = create_tmp_var(TREE_TYPE(ops[0]), "T7");
	stmt = gimple_build_assign(temp[7], BIT_NOT_EXPR, temp[6]);
	gsi_insert_after(gsi, stmt, GSI_NEW_STMT);
	/* T8 = T5 | T7 */
	temp[8] = create_tmp_var(TREE_TYPE(ops[0]), "T8");
	stmt = gimple_build_assign(temp[8], BIT_IOR_EXPR, temp[5], temp[7]);
	gsi_insert_after(gsi, stmt, GSI_NEW_STMT);
	/* T9 = ~T8 */
	temp[9] = create_tmp_var(TREE_TYPE(ops[0]), "T9");
	stmt = gimple_build_assign(temp[9], BIT_NOT_EXPR, temp[8]);
	gsi_insert_after(gsi, stmt, GSI_NEW_STMT);
	stmt = gimple_build_assign(ops[0], VAR_DECL, temp[9]);
	gsi_insert_after(gsi, stmt, GSI_NEW_STMT);

	gsi_remove(&old, true);

#ifdef DEBUG
	puts("xor obfuscated with noxor_nor");
#endif
}

static void obfuscate(gimple_stmt_iterator *gsi, gimple *xor_stmt)
{
	tree ops[3];

	/* We'll have 3 operand:
	* it's a simple assign with a BIT_XOR_EXPR as rhs.
	* on something like D.3602 = D.3601 ^ 42 we'll have
	* op0 = D.3602	(SSA_NAME)
	* op1 = D.3601	(SSA_NAME)
	* op2 = 42	(INTEGER_CST)
	*/
	for (unsigned i = 0; i < gimple_num_ops(xor_stmt); ++i)
		ops[i] = gimple_op(xor_stmt, i);

	noxor[(int)RAND(0, _nb_flavour - 1)](gsi, ops);
#ifdef DEBUG
	++xor_removed;
#endif
}

static unsigned int noxor_exec(void)
{
	gimple* stmt;
	basic_block bb;
	gimple_stmt_iterator gsi;

	if (!nofilter && !is_noxorfunc(current_function_decl))
		return 0;

	FOR_EACH_BB_FN(bb, cfun) {
		for(gsi=gsi_start_bb(bb); !gsi_end_p(gsi); gsi_next(&gsi)) {
			stmt = gsi_stmt(gsi);
			if (!(stmt && is_gimple_assign (stmt)))
				continue;

			if (gimple_assign_rhs_code (stmt) == BIT_XOR_EXPR)
				obfuscate(&gsi, stmt);
		}
	}

	return 0;
}

const pass_data pass_data_noxor = 
{
		GIMPLE_PASS,	/* Type */
		"noxor",	/* Name */
		0,		/* -fopt-info optimization group flags */
		TV_NONE,	/* Time var id */
		0,		/* prop required */
		0,		/* prop provided */
		0,		/* prop destroyed */
		0,		/* todo start flag */
		TODO_update_ssa|TODO_verify_il|TODO_cleanup_cfg,
};

class pass_noxor: public gimple_opt_pass
{
public:
	pass_noxor() : gimple_opt_pass(pass_data_noxor, NULL) {;}
	unsigned int execute(function* f) override {
		unsigned int res = noxor_exec();
#ifdef DEBUG
		if (xor_removed) {
			printf("%d xor obfuscated in function at %s:%d\n",
			       xor_removed,
			       DECL_SOURCE_FILE(current_function_decl),
			       DECL_SOURCE_LINE(current_function_decl));
			xor_removed = 0;
		}
#endif
		return res;
	}
};

static void parse_arg(int argc, plugin_argument *argv)
{
	for (int i = 0; i < argc; ++i) {
		if (strncmp("all", argv[i].key, 3) == 0)
			nofilter = true;
	}
}

int plugin_init(struct plugin_name_args *plugin_info,
		struct plugin_gcc_version *version)
{
	struct register_pass_info pass_info;
	const char *plugin_name = plugin_info->base_name;
	nofilter = false;
	noxor[mixed] = noxor_mixed;
	noxor[nand] = noxor_nand;
	noxor[nor] = noxor_nor;

	srand(time(NULL));
	pass_info.pass = new pass_noxor;
	pass_info.reference_pass_name = "ssa";
	pass_info.ref_pass_instance_number = 1;
	pass_info.pos_op = PASS_POS_INSERT_AFTER;

	register_callback(plugin_name, PLUGIN_ATTRIBUTES, register_attributes,
			  NULL);
	register_callback(plugin_name, PLUGIN_PASS_MANAGER_SETUP, NULL,
			  &pass_info);
	register_callback(plugin_name, PLUGIN_INFO, NULL, &noxor_info);
	parse_arg(plugin_info->argc, plugin_info->argv);

	return 0;
}