Minecraft server manager that automatically shuts down / restarts server for inactivity.

mcmanager.py

import asyncio
import argparse
import logging
import json
from copy import copy
from typing import Optional, Iterable

logger = logging.getLogger(__name__)

# optional setup for optimizations
try:
    import coloredlogs
    coloredlogs.install(level='INFO', fmt='%(levelname)s: %(message)s')
except ImportError:
    logger.warning('No coloredlogs package installed, use plain logger')

try:
    import uvloop
    uvloop.install()
except ImportError:
    logger.warning('uvloop package not detected, falling back to plain asyncio.'
                   ' uvloop is recommended since it has much better performance than vanilla asyncio.')


class MinecraftWrapper:
    def __init__(self, java_args, mc_args):
        self._java_args: Iterable[str] = java_args
        self._mc_args: Iterable[str] = mc_args
        self._start_event: asyncio.Event = asyncio.Event()
        self._mc_process: Optional[asyncio.subprocess.Process] = None

    async def _print_logs(self):
        # forever loop
        while not self._mc_process.stdout.at_eof():
            response = (await asyncio.create_task(self._mc_process.stdout.readline())).decode().strip()
            if 'WARN' in response:
                logger.warning(response)
            elif 'ERROR' in response:
                logger.error(response)
            else:
                logger.info(response)
            if 'Done' in response:
                # wake up start method for success
                self._start_event.set()

        # wake up start method for failure
        self._start_event.set()
        self._mc_process = None

    async def start(self):
        # if there is already a process starting, simply wait for it to finish
        if self._mc_process:
            await self._start_event.wait()
            return self._mc_process is not None

        logger.info('Starting minecraft processes')
        self._mc_process = await asyncio.create_subprocess_exec(
            'java', *[*self._java_args, '-jar', *self._mc_args],
            stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE)

        self._start_event.clear()

        # schedule the log printer task
        asyncio.create_task(self._print_logs())

        # wait for start event - either success or fail
        await self._start_event.wait()
        return self._mc_process is not None

    async def stop(self):
        if self._mc_process:
            self._mc_process.terminate()
            try:
                # if SIGTERM cannot gracefully kill the process in 30 seconds, send out SIGKILL
                await asyncio.wait([self._mc_process.wait()], timeout=30)
            except asyncio.TimeoutError:
                self._mc_process.kill()
                await self._mc_process.wait()

        self._mc_process = None
        return


class MinecraftProxyServer:
    def __init__(self, address, port, forward_address, forward_port, minecraft: MinecraftWrapper):
        self._server = None
        self._local = (address, port)
        self._forward = (forward_address, forward_port)
        self._connection_count = 0
        self._minecraft = minecraft
        self._writers = set()
        self._fake_response = None
        # value meanings:
        # None -> server is fully alive
        # not None but not done() -> fully alive but scheduled to shutdown
        # done() -> not alive
        self._shutdown_task = None
        self._entered_shutdown = False
        self._supported_protocols = set()
        self._motd = ''

    async def __new_connection(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter):
        self._connection_count += 1
        peername = writer.get_extra_info('peername')

        logger.info(f"New connection from {peername}, total connections: {self._connection_count}")

        # add writers to writer set for graceful shutdown
        self._writers.add(writer)
        # manages the connection
        try:
            await self.__serve_client(reader, writer)
        finally:
            self._connection_count -= 1
            logger.info(f"Connection closed: {peername}, total connections: {self._connection_count}")
            if self._connection_count == 0 and not self._shutdown_task:
                logger.info('Connection drops to 0, scheduling to shutdown the server')
                self._shutdown_task = asyncio.create_task(self._schedule_shutdown(60))
            self._writers.remove(writer)

    async def __serve_client(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter):
        # check the liveness of minecraft server
        # if it's alive, simply connect to the actual server and do proxy
        # if it's alive, but there is a scheduled task to shut it down, cancel the task
        # it's not alive
        # we need to behave like a fake server and answer handshake queries (to avoid frequent start/shutdown)
        # and only cancel the task / restart the actual server if the

        # first try to cancel the shutdown task
        if self._shutdown_task and not self._shutdown_task.done():
            if not self._entered_shutdown:
                logger.info('Cancel the shutdown task due to new connection')
                self._shutdown_task.cancel()
                self._shutdown_task = None
            else:
                logger.info('Failed to cancel the shutdown task as the process already begun, let it finish'
                            ' and we will restart it later')
                await self._shutdown_task

        cached_packet = bytearray()
        # then behave like a fake server, if the server is not alive
        if self._shutdown_task and self._shutdown_task.done():
            while not reader.at_eof():
                buffer, packet_size = await self._read_varint(reader)
                cached_packet.extend(buffer)
                packet_size_len = len(buffer)
                buffer, packet_id = await self._read_varint(reader)
                cached_packet.extend(buffer)

                # if it is handshake packet, we need to further decode the packet to see if it is status check or login
                protocol_version = -1
                if packet_id == 0:
                    buffer, protocol_version = await self._read_varint(reader)
                    cached_packet.extend(buffer)
                    cached_packet.extend(await reader.readexactly(packet_size - len(cached_packet) + packet_size_len))

                # if it's the handshake / status check packet
                if packet_id == 0 and cached_packet[-1] == 1:
                    # wait for ping packet to continue
                    buffer, packet_size = await self._read_varint(reader)
                    buffer, packet_id = await self._read_varint(reader)
                    if packet_size == 1 and packet_id == 0:
                        # if the protocol is supported by the server
                        response = self._fake_response
                        if protocol_version in self._supported_protocols:
                            response = copy(self._fake_response)
                            response['version']['protocol'] = protocol_version
                        # send back fake response
                        encoded = json.dumps(response).encode('utf-8')
                        packet = b'\x00' + self._write_varint(len(encoded)) + encoded
                        writer.write(self._write_varint(len(packet)) + packet)
                        await writer.drain()
                    else:
                        raise ValueError('Client sends an invalid packet after handshake packet')
                    cached_packet.clear()
                elif packet_id == 1:
                    cached_packet.extend(await reader.readexactly(packet_size - len(cached_packet) + packet_size_len))
                    writer.write(cached_packet)
                    await writer.drain()
                    writer.close()
                    await writer.wait_closed()
                    cached_packet.clear()
                # otherwise our fake server cannot handle it, restart minecraft and forward
                else:
                    logger.info('The client sent a packet that our lightweight server cannot handle, '
                                'restarting minecraft process')
                    cached_packet.extend(await reader.readexactly(packet_size - len(cached_packet) + packet_size_len))
                    start_task = asyncio.create_task(self._minecraft.start())
                    self._fake_response['description']['text'] = '(Warming up) ' + self._motd
                    # if the startup doesn't finish in 5 seconds, send back a packet requesting user to connect later
                    try:
                        await asyncio.wait_for(asyncio.shield(start_task), timeout=10)
                    except asyncio.TimeoutError:
                        # if the clients requests to login
                        if packet_id == 0 and cached_packet[-1] == 2:
                            logger.info('Timed out wating for the process to start, replying login fail')
                            # send back login error with information to connect again shortly
                            chat_msg = r"""{"text":"Server is still warming up, please try again shortly..."}"""
                            packet = b'\x00' + self._write_varint(len(chat_msg)) + chat_msg.encode('utf-8')
                            packet = self._write_varint(len(packet)) + packet
                            writer.write(packet)
                            await writer.drain()
                            writer.close()
                            await writer.wait_closed()
                        # then still wait for minecraft to fully start
                        await start_task

                    self._shutdown_task = None
                    break

        # the server must be fully alive to serve
        if not self._shutdown_task and not writer.is_closing():
            # after this point we assume the actual server is alive
            forward_reader, forward_writer = await asyncio.open_connection(*self._forward)
            self._writers.add(forward_writer)

            # first clear the buffer the client sent us (the packets that are beyond the abilities of fake server)
            forward_writer.write(cached_packet)
            await forward_writer.drain()
            readers = (reader, forward_reader)
            writers = (forward_writer, writer)
            tasks = [asyncio.create_task(reader.read(1024)) for reader in readers]
            try:
                while not reader.at_eof():
                    done, pending = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
                    for task in done:
                        data = await task
                        # forward the packet
                        index = tasks.index(task)
                        target_writer = writers[index]
                        target_writer.write(data)
                        await target_writer.drain()
                        # replace the finished tasks with a new one
                        tasks[index] = asyncio.ensure_future(readers[index].read(1024))
            finally:
                # cancel all the tasks we have scheduled
                for task in tasks:
                    task.cancel()
                # properly close the writers when we're done with the tasks
                try:
                    # try to close the writer to the client, which might already be closed
                    writer.close()
                    await writer.wait_closed()
                finally:
                    # close the forward writer, which should not raise any exceptions
                    forward_writer.close()
                    await forward_writer.wait_closed()
                    self._writers.remove(forward_writer)

    async def _schedule_shutdown(self, wait):
        await asyncio.sleep(wait)
        # shield the stop process from cancellation
        logger.info('Starting to shutdown the minecraft server')
        self._entered_shutdown = True
        await asyncio.shield(self._minecraft.stop())
        self._fake_response['description']['text'] = '(Hibernated) ' + self._motd
        self._entered_shutdown = False

    async def _read_varint(self, reader: asyncio.StreamReader):
        buffer = bytearray()
        # first read the packet bytes
        num_read = 0
        number = 0
        while True:
            read = (await reader.readexactly(1))[0]
            buffer.append(read)
            value = read & 0b01111111
            number |= (value << (7 * num_read))
            num_read += 1
            if num_read > 5:
                raise ValueError('VarInt in the packet is too big.')
            if read & 0b10000000 == 0:
                break
        return buffer, number

    def _write_varint(self, value):
        buffer = bytearray()
        while True:
            temp = value & 0b01111111
            value = value >> 7 if value >= 0 else (value + 0x100000000) >> 7
            if value != 0:
                temp |= 0b10000000
            buffer.append(temp)
            if value == 0:
                break
        return buffer

    async def start(self):
        is_ok = await self._minecraft.start()
        if not is_ok:
            return False

        # protocol test for major minecraft protocol version
        for protocol in (-1, 47, 107, 108, 109, 110, 210, 315, 316, 335, 338, 340, 393,
                         401, 404, 477, 480, 485, 490, 498, 573, 575, 578):
            # connect to mc server and store a fake handshake response
            reader, writer = await asyncio.open_connection(*self._forward)
            try:
                # handshake packet
                packet = b'\x00' + self._write_varint(protocol) + b'\x09127.0.0.1\x30\x39\x01'
                writer.write(self._write_varint(len(packet)) + packet)
                # followed by a request packet
                writer.write(b'\x01\x00')
                await writer.drain()
                _, packet_size = await self._read_varint(reader)
                _, packet_id = await self._read_varint(reader)
                if packet_id != 0:
                    remaining = await reader.read(1024)
                    raise ValueError(f'Server sent back an error packet: {self._write_varint(packet_id) + remaining}')
                _, string_size = await self._read_varint(reader)
                response = await reader.readexactly(string_size)
                response = json.loads(response.decode('utf-8'))
                if protocol == -1:
                    self._fake_response = response
                    self._motd = response['description']['text']
                if int(response['version']['protocol']) == protocol:
                    self._supported_protocols.add(protocol)
            finally:
                # and we're done
                writer.close()
                await writer.wait_closed()
        logger.info('Minecraft server started, now starting up proxy server')
        self._server = await asyncio.start_server(self.__new_connection, *self._local)

        logger.info(f'This server supports the following protocols: {self._supported_protocols}')

        # schedule to shutdown the server if no activity
        logger.info('Schedule to shutdown server due to inactivity')
        self._shutdown_task = asyncio.create_task(self._schedule_shutdown(60))
        return True

    async def stop(self):
        logger.info('Shutting down {}'.format(self))
        if self._server:
            self._server.close()
            await self._server.wait_closed()
            for writer in set(self._writers):
                writer.close()
                await writer.wait_closed()
        if self._entered_shutdown:
            await self._shutdown_task
        else:
            if self._shutdown_task:
                self._shutdown_task.cancel()
            await self._minecraft.stop()


def main():
    parser = argparse.ArgumentParser(description='A proxy wrapper for minecraft server with automatic suspension.')
    parser.add_argument('address', metavar='ADDRESS', help='The address to bind', type=str, nargs=1)
    parser.add_argument('port', metavar='PORT', help='The port to bind', type=int, nargs=1)
    parser.add_argument('forward_address', metavar='FORWARD_ADDR',
                        help='The address of the minecraft server to forward to.', type=str, nargs=1)
    parser.add_argument('forward_port', metavar='FORWARD_PORT',
                        help='The port of the minecraft server to forward to.', type=int, nargs=1)
    parser.add_argument('minecraft_args', help='The extra arguments to send to minecraft server when starting up',
                        type=str, nargs=1, default='')
    parser.add_argument('java_args', help='The extra arguments for jvm when starting up minecraft',
                        type=str, nargs=1, default='')
    results = parser.parse_args()

    loop = asyncio.get_event_loop()
    minecraft = MinecraftWrapper(results.java_args[0].split(), results.minecraft_args[0].split())
    server = MinecraftProxyServer(results.address[0], results.port[0],
                                  results.forward_address[0], results.forward_port[0], minecraft)

    try:
        is_ok = loop.run_until_complete(server.start())
        if is_ok:
            loop.run_forever()
        else:
            logger.error('Server start failed!')
    except KeyboardInterrupt:
        pass
    finally:
        loop.run_until_complete(server.stop())
    loop.close()
    return 0


if __name__ == '__main__':
    exit(main())

Problem

I'm hosting a minecraft server on my homeserver (part of this project), open to a bunch of friends who are not frequently online (i.e., the server remains idle for most of the time). However, the vanilla / spigot / paper servers all suffer from high CPU usage when idling (20%-30% on my low-end homeserver), let alone consuming a decent amount of RAM. So I wrote this manager script to automatically shut down the server when idle for some time and automatically restart it when someone tries to connect.

Features

• Lightweight: Simple python proxy server using asyncio for asynchronous connections, uvloop can optionally be used for better performance. This is a lot better than a thread-based solution.

• Automatic Shutdown and Restart: Schedule to shutdown the server when idling for a period of time and automatically restart the server when needed.

• Respond to Server List Ping: When our minecraft server is shut down due to inactivity, we should still answer the server list ping packets that the client sends us, so that the clients still see our server as alive. This is done by implementing the server list ping protocol in our proxy server, the actual minecraft server is only restarted if a client tries to log in.

Additionally, a flag (Hibernated or Warming up) is prepended to the server's description for better understanding; a friendly message indicating that server is still warming up is sent to clients when the login timed out.

Usage

You need python3.7+ to run this script

usage: mcmanager.py [-h]
                    ADDRESS PORT FORWARD_ADDR FORWARD_PORT minecraft_args
                    java_args

A proxy wrapper for minecraft server with automatic suspension.

positional arguments:
  ADDRESS         The address to bind
  PORT            The port to bind
  FORWARD_ADDR    The address of the minecraft server to forward to.
  FORWARD_PORT    The port of the minecraft server to forward to.
  minecraft_args  The extra arguments to send to minecraft server when
                  starting up
  java_args       The extra arguments for jvm when starting up minecraft

optional arguments:
  -h, --help      show this help message and exit

A typical usage would be (assuming you want to host at 192.168.1.1:25565 and the actual minecraft server is hosted at 127.0.0.1:25564):

python3 mcmanager.py 192.168.1.1 25565 127.0.0.1 25564 "<YOUR_MINECRAFT_SERVER_JAR_FILE> nogui --noconsole" "-Xmx1024M -Xms512M"