A quick and dirty server to demonstrate handling multiple protocols on the one port (uses cqueues)

multi-protocol.lua

local ctx do
	local key = require "openssl.pkey".new()
	local crt = require "openssl.x509".new()
	crt:setPublicKey(key)
	crt:sign(key)
	ctx = require "openssl.ssl.context".new("TLSv1_2", true)
	assert(ctx:setPrivateKey(key))
	assert(ctx:setCertificate(crt))
end

local cs = require "cqueues.socket"
for client in cs.listen{host="localhost", port=1234}:clients() do
	local start = ""
	::top::
	start = start .. assert(client:read(-1024))
	if start:match("^\x16\x03...\x01") then
		-- TLS Client Hello
		client:unget(start)
		start = ""
		client:starttls(ctx)
		goto top
	elseif start:match("^%w+ %S+ HTTP/1%.[01]\r\n") then
		print("HTTP")
	elseif start:match("^<.*:stream.*%sxmlns%s*=%s*(['\"])jabber:server%1.*>") then
		print("XMPP s2s")
	elseif start:match("<.*:stream.*%sxmlns%s*=%s*(['\"])jabber:client%1.*>") then
		print("XMPP c2s")
	elseif start:match("^\5") then
		print("SOCKS5")
	elseif #start == 1024 then
		error("Unknown protocol")
	else
		goto top
	end
end

Nice. Always wanted to try something like this, especially detecting the TLS handshake.