Run Windows Console Program Without Console

runsilent.c

/* clang -Wl,/subsystem:windows -Wl,/entry:mainCRTStartup -D_CRT_SECURE_NO_WARNINGS runsilent.c -luser32 -o runsilent.exe */

#include <stdlib.h>
#include <stdio.h>
#include <string.h>

#define WIN32_LEAN_AND_MEAN
#include <windows.h>

/*
 * NOTE: This program EXPLICITLY uses -A APIs for simplicity purpose.
 * Manifest this program so it uses UTF-8 if necessary or just set UTF-8 system wide!
 */

int isVerboseMode()
{
	/* The environment name is counter-intuitive */
	char *result = getenv("SILENT_VERBOSE");
	return result && strcmp(result, "1") == 0;
}

const char *getArguments(const char *argv0)
{
	const char *arg = GetCommandLineA();
	int quotes = arg[0] == '"';

	/* What the hell */
	const char *search = arg + strlen(argv0) + quotes * 2;
	while (*search == ' ')
	{
		search++;
		if (*search == 0)
			break;
	}

	return search;
}

void usage(const char *argv0)
{
	const char usageText[] = "Usage: %s <program> [arg1] [arg2] ... [argn]";

	char *buffer = malloc(sizeof(usageText) + strlen(argv0) + 1);
	sprintf(buffer, usageText, argv0);

	MessageBoxA(NULL, buffer, argv0, MB_ICONWARNING);
	free(buffer);
}

void showError(unsigned int hresult, const char *prepend)
{
	char *buffer = malloc(32 + strlen(prepend));
	sprintf(buffer, "%s\nHRESULT: 0x%08X", prepend, hresult);

	MessageBoxA(NULL, buffer, "Error", MB_ICONERROR);
	free(buffer);
}

HANDLE getHandleBase(int verbose, const char *env, int writing, const char *errprepend)
{
	const char *path = getenv(env);
	HANDLE result = INVALID_HANDLE_VALUE;
	DWORD disposition, mode;
	SECURITY_ATTRIBUTES secAttr;

	memset(&secAttr, 0, sizeof(SECURITY_ATTRIBUTES));
    secAttr.nLength = sizeof(SECURITY_ATTRIBUTES);
    secAttr.lpSecurityDescriptor = NULL;
    secAttr.bInheritHandle = 1;    

	if (writing)
	{
		mode = GENERIC_WRITE;
		disposition = CREATE_ALWAYS;
	}
	else
	{
		mode = GENERIC_READ;
		disposition = OPEN_EXISTING;
	}

	if (path)
	{
		result = CreateFileA(path, mode, 0, &secAttr, disposition, 0, NULL);

		if (result == INVALID_HANDLE_VALUE)
		{
			if (verbose)
				showError(HRESULT_FROM_WIN32(GetLastError()), errprepend);
		}
		else
			return result;
	}

	result = CreateFileA("\\\\.\\NUL", mode, 0, &secAttr, disposition, 0, NULL);
	return result;
}

HANDLE getStdin(int verbose)
{
	HANDLE result = getHandleBase(verbose, "SILENT_STDIN", 0, "Unable to open standard input.");
	if (result == INVALID_HANDLE_VALUE)
		return GetStdHandle(STD_INPUT_HANDLE);
	return result;
}

HANDLE getStdout(int verbose)
{
	HANDLE result = getHandleBase(verbose, "SILENT_STDOUT", 1, "Unable to open standard output.");
	if (result == INVALID_HANDLE_VALUE)
		return GetStdHandle(STD_OUTPUT_HANDLE);
	return result;
}

HANDLE getStderr(int verbose)
{
	HANDLE result = getHandleBase(verbose, "SILENT_STDERR", 1, "Unable to open standard error.");
	if (result == INVALID_HANDLE_VALUE)
	{
		showError(HRESULT_FROM_WIN32(GetLastError()), "stderr");
		return GetStdHandle(STD_ERROR_HANDLE);
	}
	return result;
}

int main(int argc, char *argv[])
{
	int verbose = isVerboseMode();

	if (argc < 2)
	{
		if (verbose)
			usage(argv[0]);

		return 1;
	}

	STARTUPINFOA startupInfo;
	PROCESS_INFORMATION procInfo;

	memset(&procInfo, 0, sizeof(PROCESS_INFORMATION));
	memset(&startupInfo, 0, sizeof(STARTUPINFOA));
	startupInfo.cb = sizeof(STARTUPINFOA);
	startupInfo.dwFlags = STARTF_USESTDHANDLES;
	startupInfo.hStdInput = getStdin(verbose);
	startupInfo.hStdOutput = getStdout(verbose);
	startupInfo.hStdError = getStderr(verbose);

	const char *args = getArguments(argv[0]);
	size_t argsLen = strlen(args);
	char *argsWritable = calloc(argsLen + 1, sizeof(char));
	memcpy(argsWritable, args, argsLen);

	if (verbose)
		MessageBoxA(NULL, argsWritable, "DEBUG", 0);

	int result = CreateProcessA(
		NULL,
		argsWritable,
		NULL,
		NULL,
		TRUE,
		DETACHED_PROCESS,
		NULL,
		NULL,
		&startupInfo,
		&procInfo
	);

	if (!result)
	{
		if (verbose)
			showError(HRESULT_FROM_WIN32(GetLastError()), args);

		return 1;
	}

	DWORD code = 1;
	free(argsWritable);
	WaitForSingleObject(procInfo.hProcess, INFINITE);
	GetExitCodeProcess(procInfo.hProcess, &code);
	CloseHandle(procInfo.hProcess);
	CloseHandle(procInfo.hThread);
	CloseHandle(startupInfo.hStdInput);
	CloseHandle(startupInfo.hStdOutput);
	CloseHandle(startupInfo.hStdError);

	return (int) code;
}