Win32 SChannel DTLS example

readme.txt

This program demonstrates my incorrect use of the DTLS support in the SChannel SSPI on Windows 10.

It does this in a test environment running client and server alternately on a single thread, passing messages directly between them without using a socket.

The example fails after I receive SEC_I_MESSAGE_FRAGMENT from AcceptSecurityContext (ASC). Presumably I am not successfully indicating to ASC that I want it to give me the next fragment, since it responds with SEC_E_INCOMPLETE_MESSAGE (suggesting it is expecting input data) and nothing in the output token buffer.

Example output from this program can be found in example_output.txt

dtls.cpp

static_assert(sizeof(void *) == 8, "Please build for x64.");

#pragma comment(lib, "crypt32.lib")
#pragma comment(lib, "Secur32.lib")

#include <assert.h>
#include <stdio.h>
#include <list>

#include <windows.h>
#include <schannel.h>
#include <wincrypt.h>

#define SECURITY_WIN32
#include <sspi.h>

struct Buffer
{
	char buffer[2048] = {};

	size_t start = 0;
	size_t first_after_end = 0;

	size_t length() const
	{
		assert(start <= first_after_end);
		assert(first_after_end <= ARRAYSIZE(buffer));
		return (first_after_end - start);
	}

	size_t capacity() const
	{
		assert(start <= ARRAYSIZE(buffer));
		return (ARRAYSIZE(buffer) - start);
	}

	char const *data() const
	{
		assert(start <= ARRAYSIZE(buffer));
		return &buffer[start];
	}

	char *data()
	{
		assert(start <= ARRAYSIZE(buffer));
		return &buffer[start];
	}

	bool is_in_buffer(void const *ptr) const
	{
		char const *ptr_data = static_cast<char const *>(ptr);

		if (ptr_data < buffer) return false;

		size_t offset = ptr_data - buffer;

		return (offset < sizeof(buffer));
	}

	void set_start(void const *ptr)
	{
		char const *ptr_data = static_cast<char const *>(ptr);

		assert(buffer <= ptr_data);

		size_t offset = ptr_data - buffer;

		assert(offset <= ARRAYSIZE(buffer));

		start = offset;
	}

	void set_length(size_t length)
	{
		size_t new_first_after_end = start + length;

		assert(new_first_after_end <= ARRAYSIZE(buffer));

		first_after_end = new_first_after_end;
	}
};

struct FakeConnection
{
	std::list<Buffer> client_to_server_messages;
	std::list<Buffer> server_to_client_messages;
};

struct DtlsContext
{
	bool          initialized = false;
	CtxtHandle    handle;
	unsigned long attrs;
	TimeStamp     expiry;
};

enum class HandshakeStatus
{
	INIT,
	WAITING,
	DONE,
	FAILED
};

bool create_credential(BOOL outbound, CredHandle &server_creds);
void print_message_queues(FakeConnection const &connection);
void print_buffers(char const *prefix, SecBufferDesc const &buffer_desc);
void md5(char(&out)[33], void const *data, size_t const bytes);
char const *sec_buffer_type_to_string(unsigned long type);
char const *security_hresult_to_string(HRESULT result);

HandshakeStatus ClientHandshake(CredHandle &creds, FakeConnection &connection, DtlsContext &context);
HandshakeStatus ServerHandshake(CredHandle &creds, Buffer &client_addr, FakeConnection &connection, DtlsContext &context);

int main()
{
	CredHandle server_creds;
	if (!create_credential(false, server_creds))
	{
		return 1;
	}

	CredHandle client_creds;
	if (!create_credential(true, client_creds))
	{
		return 1;
	}

	Buffer client_addr_buffer;
	{
		SOCKADDR_IN *client_addr = reinterpret_cast<SOCKADDR_IN *>(client_addr_buffer.data());

		memset(client_addr, 0, sizeof(SOCKADDR_IN));

		client_addr->sin_family = AF_INET;
		client_addr->sin_port = 443;
		client_addr->sin_addr.S_un.S_addr = 0x7f000001;

		client_addr_buffer.set_length(sizeof(SOCKADDR_IN));
	}

	FakeConnection connection;

	DtlsContext client_context;
	DtlsContext server_context;

	HandshakeStatus client_status = HandshakeStatus::INIT;
	HandshakeStatus server_status = HandshakeStatus::INIT;

	while (client_status != HandshakeStatus::DONE || server_status != HandshakeStatus::DONE)
	{
		if (client_status != HandshakeStatus::DONE)
		{
			client_status = ClientHandshake(client_creds, connection, client_context);
			if (client_status == HandshakeStatus::FAILED)
			{
				return 1;
			}
		}

		printf("\n");
		print_message_queues(connection);
		printf("\n");

		if (server_status != HandshakeStatus::DONE)
		{
			server_status = ServerHandshake(server_creds, client_addr_buffer, connection, server_context);
			if (server_status == HandshakeStatus::FAILED)
			{
				return 1;
			}
		}

		printf("\n");
		print_message_queues(connection);
		printf("\n");
	}

	printf("Handshake done!\n");

	return 0;
}

HandshakeStatus ClientHandshake(CredHandle &creds, FakeConnection &connection, DtlsContext &context)
{
	printf("Processing client handshake:\n");

	unsigned long context_reqs = ISC_REQ_CONFIDENTIALITY | ISC_REQ_EXTENDED_ERROR | ISC_REQ_DATAGRAM;

	SECURITY_STATUS isc_status = S_OK;

	while (true)
	{
		SecBufferDesc *isc_input_buffers = nullptr;

		// stack memory allocation to back (optional) isc_input_buffers
		Buffer input_buffer;
		SecBufferDesc in_buffer_desc;
		SecBuffer in_buffers[2];

		if (!context.initialized)
		{
			printf("- Context not initialised, so don't provide input yet.\n");
		}
		else if (isc_status == SEC_I_MESSAGE_FRAGMENT)
		{
			printf("- Last output was a fragment, so don't provide more input until we get all the output fragments.\n");
		}
		else
		{
			printf("- Reading input:\n");

			if (connection.server_to_client_messages.size())
			{
				input_buffer = connection.server_to_client_messages.front();
				connection.server_to_client_messages.pop_front();
				printf("  - Read next input, %llu bytes.\n", input_buffer.length());

				in_buffers[0] =
				{
					static_cast<unsigned long>(input_buffer.length()),
					SECBUFFER_TOKEN,
					input_buffer.length() ? input_buffer.data() : nullptr
				};

				in_buffers[1] =
				{
					0,
					SECBUFFER_EMPTY,
					nullptr
				};

				in_buffer_desc =
				{
					SECBUFFER_VERSION,
					ARRAYSIZE(in_buffers),
					in_buffers
				};

				printf("- Prepared input buffers:\n");
				print_buffers("  - ", in_buffer_desc);

				isc_input_buffers = &in_buffer_desc;
			}
			else
			{
				printf("  - No input available, wait for server.\n");
				return HandshakeStatus::WAITING;
			}
		}

		Buffer output_buffer, alert_buffer;

		SecBuffer out_buffers[2];

		out_buffers[0] =
		{
			static_cast<unsigned long>(output_buffer.capacity()),
			SECBUFFER_TOKEN,
			output_buffer.data()
		};

		out_buffers[1] =
		{
			static_cast<unsigned long>(alert_buffer.capacity()),
			SECBUFFER_ALERT,
			alert_buffer.data()
		};

		SecBufferDesc out_buffer_desc =
		{
			SECBUFFER_VERSION,
			ARRAYSIZE(out_buffers),
			out_buffers
		};

		printf("- Prepared output buffers:\n");
		print_buffers("  - ", out_buffer_desc);

		isc_status = InitializeSecurityContextW(
			&creds,
			context.initialized ? &context.handle : nullptr,
			nullptr,
			context_reqs,
			0,
			SECURITY_NATIVE_DREP,
			isc_input_buffers,
			0,
			context.initialized ? nullptr : &context.handle,
			&out_buffer_desc,
			&context.attrs,
			&context.expiry
		);
		context.initialized = true;

		printf("- ISC returned 0x%x: %s\n", isc_status, security_hresult_to_string(isc_status));

		if (isc_input_buffers)
		{
			printf("- Post-call input buffers:\n");
			print_buffers("  - ", *isc_input_buffers);
		}

		printf("- Post-call output buffers:\n");
		print_buffers("  - ", out_buffer_desc);

		if (isc_status != SEC_E_OK &&
			isc_status != SEC_I_CONTINUE_NEEDED &&
			isc_status != SEC_I_MESSAGE_FRAGMENT)
		{
			printf(" - We weren't expecting result 0x%x, bail out.", isc_status);
			return HandshakeStatus::FAILED;
		}

		if (isc_input_buffers)
		{
			if (isc_input_buffers->pBuffers[1].BufferType == SECBUFFER_EXTRA)
			{
				input_buffer.set_start(isc_input_buffers->pBuffers[1].pvBuffer);
				input_buffer.set_length(isc_input_buffers->pBuffers[1].cbBuffer);

				printf(
					"- ISC returned a SECBUFFER_EXTRA, pushing the remaining %llu bytes back onto the *front* of the input queue.\n",
					input_buffer.length()
				);

				connection.server_to_client_messages.push_front(input_buffer);
				input_buffer = Buffer();
			}
		}

		assert(out_buffers[0].BufferType == SECBUFFER_TOKEN);
		if (out_buffers[0].cbBuffer)
		{
			output_buffer.set_length(out_buffers[0].cbBuffer);

			printf(
				"- ISC returned a SECBUFFER_TOKEN, appending a %llu byte message to the output queue.\n",
				output_buffer.length()
			);

			connection.client_to_server_messages.push_back(output_buffer);
			output_buffer = Buffer();
		}

		if (isc_status == SEC_I_MESSAGE_FRAGMENT)
		{
			printf("- Looping back around for more fragments.\n");
			continue;
		}

		if (isc_status == SEC_I_CONTINUE_NEEDED)
		{
			printf("- Looping back around to read more data.\n");
			continue;
		}

		assert(isc_status == SEC_E_OK);
		return HandshakeStatus::DONE;
	}
}

HandshakeStatus ServerHandshake(CredHandle &creds, Buffer &client_addr, FakeConnection &connection, DtlsContext &context)
{
	printf("Processing server handshake:\n");

	unsigned long context_reqs = ASC_REQ_CONFIDENTIALITY | ASC_REQ_EXTENDED_ERROR | ASC_REQ_DATAGRAM;

	SECURITY_STATUS asc_status = S_OK;

	while (true)
	{
		Buffer input_buffer;
		SecBufferDesc in_buffer_desc;
		SecBuffer in_buffers[3];

		in_buffers[0] =
		{
			0,
			SECBUFFER_EMPTY,
			nullptr
		};

		if (asc_status == SEC_I_MESSAGE_FRAGMENT)
		{
			printf("- Last output was a fragment, so don't provide more input until we get all the output fragments.\n");
		}
		else
		{
			printf("- Reading input:\n");

			if (connection.client_to_server_messages.size())
			{
				input_buffer = connection.client_to_server_messages.front();
				connection.client_to_server_messages.pop_front();
				printf("  - Read next input, %llu bytes.\n", input_buffer.length());

				in_buffers[0] =
				{
					static_cast<unsigned long>(input_buffer.length()),
					SECBUFFER_TOKEN,
					input_buffer.length() ? input_buffer.data() : nullptr
				};
			}
			else
			{
				printf("  - No input available, wait for client.\n");
				return HandshakeStatus::WAITING;
			}
		}

		in_buffers[1] =
		{
			0,
			SECBUFFER_EMPTY,
			nullptr
		};

		in_buffers[2] =
		{
			static_cast<unsigned long>(client_addr.length()),
			SECBUFFER_EXTRA,
			client_addr.length() ? client_addr.data() : nullptr
		};

		in_buffer_desc =
		{
			SECBUFFER_VERSION,
			ARRAYSIZE(in_buffers),
			in_buffers
		};

		printf("- Prepared input buffers:\n");
		print_buffers("  - ", in_buffer_desc);

		Buffer output_buffer, alert_buffer;

		SecBuffer out_buffers[2];

		out_buffers[0] =
		{
			static_cast<unsigned long>(output_buffer.capacity()),
			SECBUFFER_TOKEN,
			output_buffer.data()
		};

		out_buffers[1] =
		{
			static_cast<unsigned long>(alert_buffer.capacity()),
			SECBUFFER_ALERT,
			alert_buffer.data()
		};

		SecBufferDesc out_buffer_desc =
		{
			SECBUFFER_VERSION,
			ARRAYSIZE(out_buffers),
			out_buffers
		};

		printf("- Prepared output buffers:\n");
		print_buffers("  - ", out_buffer_desc);

		asc_status = AcceptSecurityContext(
			&creds,
			context.initialized ? &context.handle : nullptr,
			&in_buffer_desc,
			context_reqs,
			SECURITY_NATIVE_DREP,
			context.initialized ? nullptr : &context.handle,
			&out_buffer_desc,
			&context.attrs,
			&context.expiry
		);
		context.initialized = true;

		printf("- ASC returned 0x%x: %s\n", asc_status, security_hresult_to_string(asc_status));

		printf("- Post-call input buffers:\n");
		print_buffers("  - ", in_buffer_desc);

		printf("- Post-call output buffers:\n");
		print_buffers("  - ", out_buffer_desc);

		if (asc_status != SEC_E_OK &&
			asc_status != SEC_I_CONTINUE_NEEDED &&
			asc_status != SEC_I_MESSAGE_FRAGMENT)
		{
			printf("- We weren't expecting result 0x%x, bail out.", asc_status);
			return HandshakeStatus::FAILED;
		}

		if (in_buffer_desc.pBuffers[1].BufferType == SECBUFFER_EXTRA)
		{
			input_buffer.set_start(in_buffer_desc.pBuffers[1].pvBuffer);
			input_buffer.set_length(in_buffer_desc.pBuffers[1].cbBuffer);

			printf(
				"- ASC returned a SECBUFFER_EXTRA, pushing the remaining %llu bytes back onto the *front* of the input queue.\n",
				input_buffer.length()
			);

			connection.client_to_server_messages.push_front(input_buffer);
			input_buffer = Buffer();
		}

		assert(out_buffers[0].BufferType == SECBUFFER_TOKEN);
		if (out_buffers[0].cbBuffer)
		{
			output_buffer.set_length(out_buffers[0].cbBuffer);

			printf(
				"- ASC returned a SECBUFFER_TOKEN, appending a %llu byte message to the output queue.\n",
				output_buffer.length()
			);

			connection.server_to_client_messages.push_back(output_buffer);
			output_buffer = Buffer();
		}


		if (asc_status == SEC_I_MESSAGE_FRAGMENT)
		{
			printf("- Looping back around for more fragments.\n");
			continue;
		}

		if (asc_status == SEC_I_CONTINUE_NEEDED)
		{
			printf("- Looping back around to read more data.\n");
			continue;
		}

		assert(asc_status == SEC_E_OK);
		return HandshakeStatus::DONE;
	}
}

bool create_credential(BOOL outbound, CredHandle &creds)
{
	CERT_CONTEXT const *cert_context = nullptr;

	BYTE cert_name_buffer[16];
	CERT_NAME_BLOB cert_name;
	cert_name.pbData = cert_name_buffer;
	cert_name.cbData = ARRAYSIZE(cert_name_buffer);

	if (!CertStrToNameW(
		X509_ASN_ENCODING,
		L"",
		CERT_X500_NAME_STR,
		nullptr,
		cert_name.pbData,
		&cert_name.cbData,
		nullptr
	))
	{
		printf("CertStrToNameW (with %u byte output buffer) failed with error: %u", cert_name.cbData, GetLastError());
		return false;
	}

	CERT_EXTENSIONS cert_extensions = {};

	cert_context = CertCreateSelfSignCertificate(
		0,
		&cert_name,
		0,
		nullptr,
		nullptr,
		nullptr,
		nullptr,
		&cert_extensions
	);

	if (cert_context == nullptr)
	{
		printf("CertCreateSelfSignCertificate failed with error: %u", GetLastError());
		return false;
	}

	SCHANNEL_CRED schannel_cred = {};

	schannel_cred.dwVersion = SCHANNEL_CRED_VERSION;

	schannel_cred.cCreds = 1;
	schannel_cred.paCred = &cert_context;

	schannel_cred.dwFlags = SCH_CRED_MANUAL_CRED_VALIDATION;

	TimeStamp creds_expiry;

	SECURITY_STATUS Status = AcquireCredentialsHandleW(
		nullptr,
		const_cast<wchar_t *>(UNISP_NAME_W), // Why doesn't this accept a wchar_t const *?
		outbound ? SECPKG_CRED_OUTBOUND : SECPKG_CRED_INBOUND,
		nullptr,
		&schannel_cred,
		nullptr,
		nullptr,
		&creds,
		&creds_expiry
	);

	if (Status != SEC_E_OK)
	{
		printf("AcquireCredentialsHandleW failed with error: 0x%x", Status);
		CertFreeCertificateContext(cert_context);
		return false;
	}

	CertFreeCertificateContext(cert_context);
	return true;
}

void print_message_queues(FakeConnection const &connection)
{
	printf(
		"Message Queues:\n"
		"  Client -> Server (%llu messages)",
		connection.client_to_server_messages.size()
	);

	{
		bool first = true;
		for (Buffer const &buffer : connection.client_to_server_messages)
		{
			if (first)
			{
				first = false;
				printf(":");
			}
			else
			{
				printf(",");
			}

			char md5_string[33];
			md5(md5_string, reinterpret_cast<void const *>(buffer.data()), buffer.length());

			printf(" %llu bytes (%s)", buffer.length(), md5_string);
		}
	}

	printf(
		"\n"
		"  Client <- Server (%llu messages)",
		connection.server_to_client_messages.size()
	);

	{
		bool first = true;
		for (Buffer const &buffer : connection.server_to_client_messages)
		{
			if (first)
			{
				first = false;
				printf(":");
			}
			else
			{
				printf(",");
			}

			char md5_string[33];
			md5(md5_string, reinterpret_cast<void const *>(buffer.data()), buffer.length());

			printf(" %llu bytes (%s)", buffer.length(), md5_string);
		}
	}

	printf("\n");
}

void print_buffers(char const *prefix, SecBufferDesc const &buffer_desc)
{
	for (unsigned int i = 0; i < buffer_desc.cBuffers; ++i)
	{
		printf(
			"%sBuffer[%d]: type %s, size %u",
			prefix,
			i,
			sec_buffer_type_to_string(buffer_desc.pBuffers[i].BufferType),
			buffer_desc.pBuffers[i].cbBuffer
		);

		if (buffer_desc.pBuffers[i].cbBuffer)
		{
			bool zero = true;

			for (size_t j = 0; j < buffer_desc.pBuffers[i].cbBuffer; ++j)
			{
				if (static_cast<char *>(buffer_desc.pBuffers[i].pvBuffer)[j] != 0)
				{
					zero = false;
					break;
				}
			}

			if (zero)
			{
				printf(", zero");
			}
			else
			{
				char md5_string[33];
				md5(md5_string, buffer_desc.pBuffers[i].pvBuffer, buffer_desc.pBuffers[i].cbBuffer);
				printf(", md5 %s", md5_string);
			}
		}

		printf("\n");
	}
}

void md5(char(&out)[33], void const *data, size_t const bytes)
{
	memset(out, '\0', sizeof(out));
	bool result = false;

	HCRYPTPROV context = NULL;
	assert(CryptAcquireContextW(&context, NULL, NULL, PROV_RSA_AES, CRYPT_VERIFYCONTEXT));

	HCRYPTPROV hash_provider = NULL;
	assert(CryptCreateHash(context, CALG_MD5, 0, 0, &hash_provider));

	assert(CryptHashData(hash_provider, reinterpret_cast<BYTE *>(const_cast<void *>(data)), static_cast<DWORD>(bytes), 0));

	DWORD hash_size = 0;
	DWORD hash_size_param_size = sizeof(hash_size);
	assert(CryptGetHashParam(hash_provider, HP_HASHSIZE, reinterpret_cast<BYTE *>(&hash_size), &hash_size_param_size, 0));

	unsigned long long hash_value[2];
	assert(hash_size == sizeof(hash_value));

	assert(CryptGetHashParam(hash_provider, HP_HASHVAL, reinterpret_cast<BYTE *>(hash_value), &hash_size, 0));

	assert((sizeof(out) - 1) ==
		sprintf_s(out,
			"%016llX%016llX",
			hash_value[0], hash_value[1]
		)
	);

	CryptDestroyHash(hash_provider);
	CryptReleaseContext(context, 0);
}

char const *sec_buffer_type_to_string(unsigned long type)
{
	switch (type)
	{
	case SECBUFFER_EMPTY:                      return "SECBUFFER_EMPTY";
	case SECBUFFER_DATA:                       return "SECBUFFER_DATA";
	case SECBUFFER_TOKEN:                      return "SECBUFFER_TOKEN";
	case SECBUFFER_PKG_PARAMS:                 return "SECBUFFER_PKG_PARAMS";
	case SECBUFFER_MISSING:                    return "SECBUFFER_MISSING";
	case SECBUFFER_EXTRA:                      return "SECBUFFER_EXTRA";
	case SECBUFFER_STREAM_TRAILER:             return "SECBUFFER_STREAM_TRAILER";
	case SECBUFFER_STREAM_HEADER:              return "SECBUFFER_STREAM_HEADER";
	case SECBUFFER_NEGOTIATION_INFO:           return "SECBUFFER_NEGOTIATION_INFO";
	case SECBUFFER_PADDING:                    return "SECBUFFER_PADDING";
	case SECBUFFER_STREAM:                     return "SECBUFFER_STREAM";
	case SECBUFFER_MECHLIST:                   return "SECBUFFER_MECHLIST";
	case SECBUFFER_MECHLIST_SIGNATURE:         return "SECBUFFER_MECHLIST_SIGNATURE";
	case SECBUFFER_TARGET:                     return "SECBUFFER_TARGET";
	case SECBUFFER_CHANNEL_BINDINGS:           return "SECBUFFER_CHANNEL_BINDINGS";
	case SECBUFFER_CHANGE_PASS_RESPONSE:       return "SECBUFFER_CHANGE_PASS_RESPONSE";
	case SECBUFFER_TARGET_HOST:                return "SECBUFFER_TARGET_HOST";
	case SECBUFFER_ALERT:                      return "SECBUFFER_ALERT";
	case SECBUFFER_APPLICATION_PROTOCOLS:      return "SECBUFFER_APPLICATION_PROTOCOLS";
	case SECBUFFER_SRTP_PROTECTION_PROFILES:   return "SECBUFFER_SRTP_PROTECTION_PROFILES";
	case SECBUFFER_SRTP_MASTER_KEY_IDENTIFIER: return "SECBUFFER_SRTP_MASTER_KEY_IDENTIFIER";
	case SECBUFFER_TOKEN_BINDING:              return "SECBUFFER_TOKEN_BINDING";
	case SECBUFFER_PRESHARED_KEY:              return "SECBUFFER_PRESHARED_KEY";
	case SECBUFFER_PRESHARED_KEY_IDENTITY:     return "SECBUFFER_PRESHARED_KEY_IDENTITY";
	case SECBUFFER_DTLS_MTU:                   return "SECBUFFER_DTLS_MTU";
	default:                                   return "unknown";
	}
}

char const *security_hresult_to_string(HRESULT result)
{
	switch (result)
	{
	case SEC_E_OK:                            return "SEC_E_OK - The operation completed successfully.";
	case SEC_I_CONTINUE_NEEDED:               return "SEC_I_CONTINUE_NEEDED - The function completed successfully, but must be called again to complete the context";
	case SEC_I_COMPLETE_NEEDED:               return "SEC_I_COMPLETE_NEEDED - The function completed successfully, but CompleteToken must be called";
	case SEC_I_COMPLETE_AND_CONTINUE:         return "SEC_I_COMPLETE_AND_CONTINUE - The function completed successfully, but both CompleteToken and this function must be called to complete the context";
	case SEC_I_LOCAL_LOGON:                   return "SEC_I_LOCAL_LOGON - The logon was completed, but no network authority was available. The logon was made using locally known information";
	case SEC_I_CONTEXT_EXPIRED:               return "SEC_I_CONTEXT_EXPIRED - The context has expired and can no longer be used.";
	case SEC_I_INCOMPLETE_CREDENTIALS:        return "SEC_I_INCOMPLETE_CREDENTIALS - The credentials supplied were not complete, and could not be verified. Additional information can be returned from the context.";
	case SEC_I_RENEGOTIATE:                   return "SEC_I_RENEGOTIATE - The context data must be renegotiated with the peer.";
	case SEC_I_NO_LSA_CONTEXT:                return "SEC_I_NO_LSA_CONTEXT - There is no LSA mode context associated with this context.";
	case SEC_I_SIGNATURE_NEEDED:              return "SEC_I_SIGNATURE_NEEDED - A signature operation must be performed before the user can authenticate.";
	case SEC_I_NO_RENEGOTIATION:              return "SEC_I_NO_RENEGOTIATION - The recipient rejected the renegotiation request.";
	case SEC_I_MESSAGE_FRAGMENT:              return "SEC_I_MESSAGE_FRAGMENT - The returned buffer is only a fragment of the message.  More fragments need to be returned.";
	case SEC_I_CONTINUE_NEEDED_MESSAGE_OK:    return "SEC_I_CONTINUE_NEEDED_MESSAGE_OK - The function completed successfully, but must be called again to complete the context.  Early start can be used.";
	case SEC_I_ASYNC_CALL_PENDING:            return "SEC_I_ASYNC_CALL_PENDING - An asynchronous SSPI routine has been called and the work is pending completion.";
	case SEC_E_INSUFFICIENT_MEMORY:           return "SEC_E_INSUFFICIENT_MEMORY - Not enough memory is available to complete this request";
	case SEC_E_INVALID_HANDLE:                return "SEC_E_INVALID_HANDLE - The handle specified is invalid";
	case SEC_E_UNSUPPORTED_FUNCTION:          return "SEC_E_UNSUPPORTED_FUNCTION - The function requested is not supported";
	case SEC_E_TARGET_UNKNOWN:                return "SEC_E_TARGET_UNKNOWN - The specified target is unknown or unreachable";
	case SEC_E_INTERNAL_ERROR:                return "SEC_E_INTERNAL_ERROR - The Local Security Authority cannot be contacted";
	case SEC_E_SECPKG_NOT_FOUND:              return "SEC_E_SECPKG_NOT_FOUND - The requested security package does not exist";
	case SEC_E_NOT_OWNER:                     return "SEC_E_NOT_OWNER - The caller is not the owner of the desired credentials";
	case SEC_E_CANNOT_INSTALL:                return "SEC_E_CANNOT_INSTALL - The security package failed to initialize, and cannot be installed";
	case SEC_E_INVALID_TOKEN:                 return "SEC_E_INVALID_TOKEN - The token supplied to the function is invalid";
	case SEC_E_CANNOT_PACK:                   return "SEC_E_CANNOT_PACK - The security package is not able to marshall the logon buffer, so the logon attempt has failed";
	case SEC_E_QOP_NOT_SUPPORTED:             return "SEC_E_QOP_NOT_SUPPORTED - The per-message Quality of Protection is not supported by the security package";
	case SEC_E_NO_IMPERSONATION:              return "SEC_E_NO_IMPERSONATION - The security context does not allow impersonation of the client";
	case SEC_E_LOGON_DENIED:                  return "SEC_E_LOGON_DENIED - The logon attempt failed";
	case SEC_E_UNKNOWN_CREDENTIALS:           return "SEC_E_UNKNOWN_CREDENTIALS - The credentials supplied to the package were not recognized";
	case SEC_E_NO_CREDENTIALS:                return "SEC_E_NO_CREDENTIALS - No credentials are available in the security package";
	case SEC_E_MESSAGE_ALTERED:               return "SEC_E_MESSAGE_ALTERED - The message or signature supplied for verification has been altered";
	case SEC_E_OUT_OF_SEQUENCE:               return "SEC_E_OUT_OF_SEQUENCE - The message supplied for verification is out of sequence";
	case SEC_E_NO_AUTHENTICATING_AUTHORITY:   return "SEC_E_NO_AUTHENTICATING_AUTHORITY - No authority could be contacted for authentication.";
	case SEC_E_BAD_PKGID:                     return "SEC_E_BAD_PKGID - The requested security package does not exist";
	case SEC_E_CONTEXT_EXPIRED:               return "SEC_E_CONTEXT_EXPIRED - The context has expired and can no longer be used.";
	case SEC_E_INCOMPLETE_MESSAGE:            return "SEC_E_INCOMPLETE_MESSAGE - The supplied message is incomplete. The signature was not verified.";
	case SEC_E_INCOMPLETE_CREDENTIALS:        return "SEC_E_INCOMPLETE_CREDENTIALS - The credentials supplied were not complete, and could not be verified. The context could not be initialized.";
	case SEC_E_BUFFER_TOO_SMALL:              return "SEC_E_BUFFER_TOO_SMALL - The buffers supplied to a function was too small.";
	case SEC_E_WRONG_PRINCIPAL:               return "SEC_E_WRONG_PRINCIPAL - The target principal name is incorrect.";
	case SEC_E_TIME_SKEW:                     return "SEC_E_TIME_SKEW - The clocks on the client and server machines are skewed.";
	case SEC_E_UNTRUSTED_ROOT:                return "SEC_E_UNTRUSTED_ROOT - The certificate chain was issued by an authority that is not trusted.";
	case SEC_E_ILLEGAL_MESSAGE:               return "SEC_E_ILLEGAL_MESSAGE - The message received was unexpected or badly formatted.";
	case SEC_E_CERT_UNKNOWN:                  return "SEC_E_CERT_UNKNOWN - An unknown error occurred while processing the certificate.";
	case SEC_E_CERT_EXPIRED:                  return "SEC_E_CERT_EXPIRED - The received certificate has expired.";
	case SEC_E_ENCRYPT_FAILURE:               return "SEC_E_ENCRYPT_FAILURE - The specified data could not be encrypted.";
	case SEC_E_DECRYPT_FAILURE:               return "SEC_E_DECRYPT_FAILURE - The specified data could not be decrypted.";
	case SEC_E_ALGORITHM_MISMATCH:            return "SEC_E_ALGORITHM_MISMATCH - The client and server cannot communicate, because they do not possess a common algorithm.";
	case SEC_E_SECURITY_QOS_FAILED:           return "SEC_E_SECURITY_QOS_FAILED - The security context could not be established due to a failure in the requested quality of service (e.g. mutual authentication or delegation).";
	case SEC_E_UNFINISHED_CONTEXT_DELETED:    return "SEC_E_UNFINISHED_CONTEXT_DELETED - A security context was deleted before the context was completed. This is considered a logon failure.";
	case SEC_E_NO_TGT_REPLY:                  return "SEC_E_NO_TGT_REPLY - The client is trying to negotiate a context and the server requires user-to-user but didn't send a TGT reply.";
	case SEC_E_NO_IP_ADDRESSES:               return "SEC_E_NO_IP_ADDRESSES - Unable to accomplish the requested task because the local machine does not have any IP addresses.";
	case SEC_E_WRONG_CREDENTIAL_HANDLE:       return "SEC_E_WRONG_CREDENTIAL_HANDLE - The supplied credential handle does not match the credential associated with the security context.";
	case SEC_E_CRYPTO_SYSTEM_INVALID:         return "SEC_E_CRYPTO_SYSTEM_INVALID - The crypto system or checksum function is invalid because a required function is unavailable.";
	case SEC_E_MAX_REFERRALS_EXCEEDED:        return "SEC_E_MAX_REFERRALS_EXCEEDED - The number of maximum ticket referrals has been exceeded.";
	case SEC_E_MUST_BE_KDC:                   return "SEC_E_MUST_BE_KDC - The local machine must be a Kerberos KDC (domain controller) and it is not.";
	case SEC_E_STRONG_CRYPTO_NOT_SUPPORTED:   return "SEC_E_STRONG_CRYPTO_NOT_SUPPORTED - The other end of the security negotiation is requires strong crypto but it is not supported on the local machine.";
	case SEC_E_TOO_MANY_PRINCIPALS:           return "SEC_E_TOO_MANY_PRINCIPALS - The KDC reply contained more than one principal name.";
	case SEC_E_NO_PA_DATA:                    return "SEC_E_NO_PA_DATA - Expected to find PA data for a hint of what etype to use, but it was not found.";
	case SEC_E_PKINIT_NAME_MISMATCH:          return "SEC_E_PKINIT_NAME_MISMATCH - The client certificate does not contain a valid UPN, or does not match the client name in the logon request. Please contact your administrator.";
	case SEC_E_SMARTCARD_LOGON_REQUIRED:      return "SEC_E_SMARTCARD_LOGON_REQUIRED - Smartcard logon is required and was not used.";
	case SEC_E_SHUTDOWN_IN_PROGRESS:          return "SEC_E_SHUTDOWN_IN_PROGRESS - A system shutdown is in progress.";
	case SEC_E_KDC_INVALID_REQUEST:           return "SEC_E_KDC_INVALID_REQUEST - An invalid request was sent to the KDC.";
	case SEC_E_KDC_UNABLE_TO_REFER:           return "SEC_E_KDC_UNABLE_TO_REFER - The KDC was unable to generate a referral for the service requested.";
	case SEC_E_KDC_UNKNOWN_ETYPE:             return "SEC_E_KDC_UNKNOWN_ETYPE - The encryption type requested is not supported by the KDC.";
	case SEC_E_UNSUPPORTED_PREAUTH:           return "SEC_E_UNSUPPORTED_PREAUTH - An unsupported preauthentication mechanism was presented to the Kerberos package.";
	case SEC_E_DELEGATION_REQUIRED:           return "SEC_E_DELEGATION_REQUIRED - The requested operation cannot be completed. The computer must be trusted for delegation and the current user account must be configured to allow delegation.";
	case SEC_E_BAD_BINDINGS:                  return "SEC_E_BAD_BINDINGS - Client's supplied SSPI channel bindings were incorrect.";
	case SEC_E_MULTIPLE_ACCOUNTS:             return "SEC_E_MULTIPLE_ACCOUNTS - The received certificate was mapped to multiple accounts.";
	case SEC_E_NO_KERB_KEY:                   return "SEC_E_NO_KERB_KEY";
	case SEC_E_CERT_WRONG_USAGE:              return "SEC_E_CERT_WRONG_USAGE - The certificate is not valid for the requested usage.";
	case SEC_E_DOWNGRADE_DETECTED:            return "SEC_E_DOWNGRADE_DETECTED - The system cannot contact a domain controller to service the authentication request. Please try again later.";
	case SEC_E_SMARTCARD_CERT_REVOKED:        return "SEC_E_SMARTCARD_CERT_REVOKED - The smartcard certificate used for authentication has been revoked. Please contact your system administrator. There may be additional information in the event log.";
	case SEC_E_ISSUING_CA_UNTRUSTED:          return "SEC_E_ISSUING_CA_UNTRUSTED - An untrusted certificate authority was detected while processing the smartcard certificate used for authentication. Please contact your system administrator.";
	case SEC_E_REVOCATION_OFFLINE_C:          return "SEC_E_REVOCATION_OFFLINE_C - The revocation status of the smartcard certificate used for authentication could not be determined. Please contact your system administrator.";
	case SEC_E_PKINIT_CLIENT_FAILURE:         return "SEC_E_PKINIT_CLIENT_FAILURE - The smartcard certificate used for authentication was not trusted. Please contact your system administrator.";
	case SEC_E_SMARTCARD_CERT_EXPIRED:        return "SEC_E_SMARTCARD_CERT_EXPIRED - The smartcard certificate used for authentication has expired. Please contact your system administrator.";
	case SEC_E_NO_S4U_PROT_SUPPORT:           return "SEC_E_NO_S4U_PROT_SUPPORT - The Kerberos subsystem encountered an error. A service for user protocol request was made against a domain controller which does not support service for user.";
	case SEC_E_CROSSREALM_DELEGATION_FAILURE: return "SEC_E_CROSSREALM_DELEGATION_FAILURE - An attempt was made by this server to make a Kerberos constrained delegation request for a target outside of the server's realm. This is not supported, and indicates a misconfiguration on this server's allowed to delegate to list. Please contact your administrator.";
	case SEC_E_REVOCATION_OFFLINE_KDC:        return "SEC_E_REVOCATION_OFFLINE_KDC - The revocation status of the domain controller certificate used for smartcard authentication could not be determined. There is additional information in the system event log. Please contact your system administrator.";
	case SEC_E_ISSUING_CA_UNTRUSTED_KDC:      return "SEC_E_ISSUING_CA_UNTRUSTED_KDC - An untrusted certificate authority was detected while processing the domain controller certificate used for authentication. There is additional information in the system event log. Please contact your system administrator.";
	case SEC_E_KDC_CERT_EXPIRED:              return "SEC_E_KDC_CERT_EXPIRED - The domain controller certificate used for smartcard logon has expired. Please contact your system administrator with the contents of your system event log.";
	case SEC_E_KDC_CERT_REVOKED:              return "SEC_E_KDC_CERT_REVOKED - The domain controller certificate used for smartcard logon has been revoked. Please contact your system administrator with the contents of your system event log.";
	case SEC_E_INVALID_PARAMETER:             return "SEC_E_INVALID_PARAMETER - One or more of the parameters passed to the function was invalid.";
	case SEC_E_DELEGATION_POLICY:             return "SEC_E_DELEGATION_POLICY - Client policy does not allow credential delegation to target server.";
	case SEC_E_POLICY_NLTM_ONLY:              return "SEC_E_POLICY_NLTM_ONLY - Client policy does not allow credential delegation to target server with NLTM only authentication.";
	case SEC_E_NO_CONTEXT:                    return "SEC_E_NO_CONTEXT - The required security context does not exist.";
	case SEC_E_PKU2U_CERT_FAILURE:            return "SEC_E_PKU2U_CERT_FAILURE - The PKU2U protocol encountered an error while attempting to utilize the associated certificates.";
	case SEC_E_MUTUAL_AUTH_FAILED:            return "SEC_E_MUTUAL_AUTH_FAILED - The identity of the server computer could not be verified.";
	case SEC_E_ONLY_HTTPS_ALLOWED:            return "SEC_E_ONLY_HTTPS_ALLOWED - Only https scheme is allowed.";
	case SEC_E_APPLICATION_PROTOCOL_MISMATCH: return "SEC_E_APPLICATION_PROTOCOL_MISMATCH - No common application protocol exists between the client and the server. Application protocol negotiation failed.";
	case SEC_E_INVALID_UPN_NAME:              return "SEC_E_INVALID_UPN_NAME - You can't sign in with a user ID in this format. Try using your email address instead.";
	default:                                  return "unknown";
	}
}

example_output.txt

Processing client handshake:
- Context not initialised, so don't provide input yet.
- Prepared output buffers:
  - Buffer[0]: type SECBUFFER_TOKEN, size 2048, zero
  - Buffer[1]: type SECBUFFER_ALERT, size 2048, zero
- ISC returned 0x90312: SEC_I_CONTINUE_NEEDED - The function completed successfully, but must be called again to complete the context
- Post-call output buffers:
  - Buffer[0]: type SECBUFFER_TOKEN, size 170, md5 2BBB931BAE77B88537B126DC42B80AF7
  - Buffer[1]: type SECBUFFER_ALERT, size 0
- ISC returned a SECBUFFER_TOKEN, appending a 170 byte message to the output queue.
- Looping back around to read more data.
- Reading input:
  - No input available, wait for server.

Message Queues:
  Client -> Server (1 messages): 170 bytes (2BBB931BAE77B88537B126DC42B80AF7)
  Client <- Server (0 messages)

Processing server handshake:
- Reading input:
  - Read next input, 170 bytes.
- Prepared input buffers:
  - Buffer[0]: type SECBUFFER_TOKEN, size 170, md5 2BBB931BAE77B88537B126DC42B80AF7
  - Buffer[1]: type SECBUFFER_EMPTY, size 0
  - Buffer[2]: type SECBUFFER_EXTRA, size 16, md5 F13EAAAF56C42AC5C1985F609CF6809E
- Prepared output buffers:
  - Buffer[0]: type SECBUFFER_TOKEN, size 2048, zero
  - Buffer[1]: type SECBUFFER_ALERT, size 2048, zero
- ASC returned 0x90312: SEC_I_CONTINUE_NEEDED - The function completed successfully, but must be called again to complete the context
- Post-call input buffers:
  - Buffer[0]: type SECBUFFER_TOKEN, size 170, md5 2BBB931BAE77B88537B126DC42B80AF7
  - Buffer[1]: type SECBUFFER_EMPTY, size 0
  - Buffer[2]: type SECBUFFER_EXTRA, size 16, md5 F13EAAAF56C42AC5C1985F609CF6809E
- Post-call output buffers:
  - Buffer[0]: type SECBUFFER_TOKEN, size 60, md5 3AF5D8A032F2FC22C2EB4B50050007F0
  - Buffer[1]: type SECBUFFER_ALERT, size 0
- ASC returned a SECBUFFER_TOKEN, appending a 60 byte message to the output queue.
- Looping back around to read more data.
- Reading input:
  - No input available, wait for client.

Message Queues:
  Client -> Server (0 messages)
  Client <- Server (1 messages): 60 bytes (3AF5D8A032F2FC22C2EB4B50050007F0)

Processing client handshake:
- Reading input:
  - Read next input, 60 bytes.
- Prepared input buffers:
  - Buffer[0]: type SECBUFFER_TOKEN, size 60, md5 3AF5D8A032F2FC22C2EB4B50050007F0
  - Buffer[1]: type SECBUFFER_EMPTY, size 0
- Prepared output buffers:
  - Buffer[0]: type SECBUFFER_TOKEN, size 2048, zero
  - Buffer[1]: type SECBUFFER_ALERT, size 2048, zero
- ISC returned 0x90312: SEC_I_CONTINUE_NEEDED - The function completed successfully, but must be called again to complete the context
- Post-call input buffers:
  - Buffer[0]: type SECBUFFER_TOKEN, size 60, md5 3AF5D8A032F2FC22C2EB4B50050007F0
  - Buffer[1]: type SECBUFFER_EMPTY, size 0
- Post-call output buffers:
  - Buffer[0]: type SECBUFFER_TOKEN, size 202, md5 BE78D77B1B85E03D160C53D806133E4E
  - Buffer[1]: type SECBUFFER_ALERT, size 0
- ISC returned a SECBUFFER_TOKEN, appending a 202 byte message to the output queue.
- Looping back around to read more data.
- Reading input:
  - No input available, wait for server.

Message Queues:
  Client -> Server (1 messages): 202 bytes (BE78D77B1B85E03D160C53D806133E4E)
  Client <- Server (0 messages)

Processing server handshake:
- Reading input:
  - Read next input, 202 bytes.
- Prepared input buffers:
  - Buffer[0]: type SECBUFFER_TOKEN, size 202, md5 BE78D77B1B85E03D160C53D806133E4E
  - Buffer[1]: type SECBUFFER_EMPTY, size 0
  - Buffer[2]: type SECBUFFER_EXTRA, size 16, md5 F13EAAAF56C42AC5C1985F609CF6809E
- Prepared output buffers:
  - Buffer[0]: type SECBUFFER_TOKEN, size 2048, zero
  - Buffer[1]: type SECBUFFER_ALERT, size 2048, zero
- ASC returned 0x90364: SEC_I_MESSAGE_FRAGMENT - The returned buffer is only a fragment of the message.  More fragments need to be returned.
- Post-call input buffers:
  - Buffer[0]: type SECBUFFER_TOKEN, size 202, md5 BE78D77B1B85E03D160C53D806133E4E
  - Buffer[1]: type SECBUFFER_EMPTY, size 0
  - Buffer[2]: type SECBUFFER_EXTRA, size 16, md5 F13EAAAF56C42AC5C1985F609CF6809E
- Post-call output buffers:
  - Buffer[0]: type SECBUFFER_TOKEN, size 759, md5 CED73575D6EDDD574FD5547AFC3A7E62
  - Buffer[1]: type SECBUFFER_ALERT, size 0
- ASC returned a SECBUFFER_TOKEN, appending a 759 byte message to the output queue.
- Looping back around for more fragments.
- Last output was a fragment, so don't provide more input until we get all the output fragments.
- Prepared input buffers:
  - Buffer[0]: type SECBUFFER_EMPTY, size 0
  - Buffer[1]: type SECBUFFER_EMPTY, size 0
  - Buffer[2]: type SECBUFFER_EXTRA, size 16, md5 F13EAAAF56C42AC5C1985F609CF6809E
- Prepared output buffers:
  - Buffer[0]: type SECBUFFER_TOKEN, size 2048, zero
  - Buffer[1]: type SECBUFFER_ALERT, size 2048, zero
- ASC returned 0x80090318: SEC_E_INCOMPLETE_MESSAGE - The supplied message is incomplete. The signature was not verified.
- Post-call input buffers:
  - Buffer[0]: type SECBUFFER_EMPTY, size 0
  - Buffer[1]: type SECBUFFER_EMPTY, size 0
  - Buffer[2]: type SECBUFFER_EXTRA, size 16, md5 F13EAAAF56C42AC5C1985F609CF6809E
- Post-call output buffers:
  - Buffer[0]: type SECBUFFER_TOKEN, size 2048, zero
  - Buffer[1]: type SECBUFFER_ALERT, size 0
- We weren't expecting result 0x80090318, bail out.

According to my test, the only WRONG thing you did is forget to RESET context.handle once initialized during handshaking.

isc_status = InitializeSecurityContextW(
    &creds,
    context.initialized ? &context.handle : nullptr,
    nullptr,
    context_reqs,
    0,
    SECURITY_NATIVE_DREP,
    isc_input_buffers,
    0,
    &context.handle, // HERE
    &out_buffer_desc,
    &context.attrs,
    &context.expiry
);

asc_status = AcceptSecurityContext(
    &creds,
    context.initialized ? &context.handle : nullptr,
    &in_buffer_desc,
    context_reqs,
    SECURITY_NATIVE_DREP,
    &context.handle, // HERE
    &out_buffer_desc,
    &context.attrs,
    &context.expiry
);

It's really painful to debug this without any docs describing it...

Hi Pu Xu, thanks for your reply, looks like that was exactly it.

It seems I was mislead by this line from the docs for InitializeSecurityContextW:

When using the Schannel SSP, on calls after the first call, pass the handle returned here as the phContext parameter and specify NULL for phNewContext.

If you're interested in claiming credit for solving my problem, I have open questions on StackOverflow and also MS Docs Q&A. If not, I'll answer those questions for the benefit of anyone who finds them.

Thanks again,
Haddon.