GameMaker network handshake implementation.

GMHandshake.cs

#nullable enable

using System;
using System.Net.Sockets;
using System.Runtime.Serialization;
using System.Text;

namespace nkrapivindev
{
	/// <summary>
	/// When something bad happens during the GameMaker handshake.
	/// </summary>
	public class GMHandshakeException : Exception
	{
		/// <summary>
		/// Was the GameMaker side a server, or not?
		/// </summary>
		public bool IsServer { get; set; }

		public GMHandshakeException() : base() { }
		public GMHandshakeException(string? message) : base(message) { }
		public GMHandshakeException(string? message, Exception? innerException) : base(message, innerException) { }
		protected GMHandshakeException(SerializationInfo info, StreamingContext context) : base(info, context) { }
		public GMHandshakeException(string message, bool isServer) : base(message) { IsServer = isServer; }
	}

	/// <summary>
	/// Contains the extension method for establishing a handshake with GameMaker.
	/// </summary>
	public static class GMHandshake
	{
		/// <summary>
		/// Performs a handshake with the GameMaker side.
		/// </summary>
		/// <param name="isServer">connecting to a server?</param>
		public static void PerformGMSHandshake(this Socket self, bool isServer = true)
		{
			/* Usage:
			 * Socket s = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
			 * sock.Connect("127.0.0.1", 6969);
			 * sock.PerformGMSHandshake(true); // must be done right after .Connect()
			 * and now try to send something:
			 * sock.Send(Encoding.ASCII.GetBytes("hello!"));
			 * sock.Receive(new byte[1]);
			 * 
			 * an async - networking event should pop up with the `hello!` buffer!
			 */

			int recv; // bytes received
			int send; // bytes sent

			// stage 1 stuff
			const string message = "GM:Studio-Connect\0";
			// FYI C# string are not null terminated, be careful. the nullbyte must be present at the end.
			// and is a part of the data size!

			// uint - 32 bit unsigned integer type.

			// stage 2 stuff
			// struct SYYLoginPacket {
			//   uint SIG1 = 0xcafebabe;
			//   uint SIG2 = 0xdeadb00b;
			//   uint SIZE = 0x10;
			//   uint WEIRD; // seems to be corrupted memory imo. can be zero or whatever.
			// };
			// here byte endianess matters (I think?):
			byte[] stage2 = { 0xbe, 0xba, 0xfe, 0xca, 0x0b, 0xb0, 0xad, 0xde, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 };

			// stage 2 client stuff
			// struct SYYLoginPacketClient {
			//   uint SIG1 = 0xdeafbead;
			//   uint SIG2 = 0xf00dbeeb;
			//   uint SIZE = 0xc;
			// };
			byte[] stage2c = { 0xad, 0xbe, 0xaf, 0xde, 0xeb, 0xbe, 0x0d, 0xf0, 0x0c, 0x00, 0x00, 0x00 };

			if (isServer) // we're connecting to a network_create_server server.
			{
				// stage 1 [receive]: Hello, dear GameMaker!
				byte[] asciibuf = new byte[18];
				recv = self.Receive(asciibuf, 0, 18, SocketFlags.None);
				if (recv != 18)
				{
					throw new GMHandshakeException($"Handshake stage 1 failed. Did not receive enough bytes, expected 18 got {recv}.", isServer);
				}

				// stage 1: verify
				string str = Encoding.ASCII.GetString(asciibuf);
				if (str != message)
				{
					throw new GMHandshakeException($"Handshake stage 1 failed. Invalid Hello message, expected '{message}', got '{str}'.", isServer);
				}

				// stage 2 [send]: Dead boob, cafe babe.
				send = self.Send(stage2, 0, 16, SocketFlags.None);
				if (send != 16)
				{
					throw new GMHandshakeException($"Handshake stage 2 failed. Send failed, sent {send} bytes instead of 16.", isServer);
				}

				// stage 2: verify
				byte[] stage2creply = new byte[12];
				recv = self.Receive(stage2creply, 0, 12, SocketFlags.None);
				if (recv != 12)
				{
					throw new GMHandshakeException($"Handshake stage 2 failed. Receive failed, unable to receive stage 2 reply, {recv} instead of 12.", isServer);
				}

				for (int i = 0; i < 12; ++i)
				{
					if (stage2c[i] != stage2creply[i])
						throw new GMHandshakeException($"Handshake stage 2 failed. Invalid stage 2 reply, corrupted data at index {i}.", isServer);
				}
			}
			else // we're connecting to a network_create_socket socket.
			{
				// stage 1 [send]: Hello, dear GameMaker!
				byte[] asciibuf = Encoding.ASCII.GetBytes(message);
				send = self.Send(asciibuf, 0, 18, SocketFlags.None);
				if (send != 18)
				{
					throw new GMHandshakeException($"Handshake stage 1 failed. Did not send the whole string, only {send} out of 18 bytes.", isServer);
				}

				// stage 1 is verified by GameMaker.

				// stage 2 [receive]: Dead boob, cafe babe.
				byte[] cafebabe = new byte[16];
				recv = self.Receive(cafebabe);
				if (recv != 16)
				{
					throw new GMHandshakeException($"Handshake stage 2 failed. Did not receive enough bytes, expected 16 got {recv}.", isServer);
				}

				// only compare first 12 bytes of the packet, ignoring the WEIRD field.
				for (int i = 0; i < 12; ++i)
				{
					if (cafebabe[i] != stage2[i])
						throw new GMHandshakeException($"Handshake stage 2 failed. Invalid Cafebabe packet, corrupted at index {i}.", isServer);
				}

				// reply to stage 2
				send = self.Send(stage2c, 0, 12, SocketFlags.None);
				if (send != 12)
				{
					throw new GMHandshakeException($"Handshake stage 2 failed. Failed to reply to a stage 2 packet, sent {send} instead of 12.", isServer);
				}

				// yoyo debil games, hackfix for GM not noticing the stage 2 response.
				int oldtimeout = self.ReceiveTimeout;
				self.ReceiveTimeout = 1; // lowest possible timeout, 1ms.
				try
				{
					self.Receive(new byte[1]);
				}
				catch { }
				// after hitting the timeout intentionally, we can restore the timeout back.
				self.ReceiveTimeout = oldtimeout;
			}

			// if we've reached this point, we're done! an async event should pop up in GM.
		}
	}
}