Packet handler used to generate most of BedrockData for PocketMine-MP (PM3 only)

README.md

This script operates on a packet dump between a vanilla client and server.

A dump like this can be obtained in various ways - a proxy, my frida tracer, etc. However you get it, the file provided should have one packet per line, starting with read: or write: and ending with a base64-encoded packet (including the packet ID).

A quick example grabbed from one of my old log files:

read:kAHwoMpB4PUgQQAAAD8g17NAAAAAPwAAAAAAAAAA4PUgQQABAg==
write:CQ==
write:CQIBG8KnZSVtdWx0aXBsYXllci5wbGF5ZXIubGVmdAELbWN0ZXN0RHlsYW4AAA==

Run this script in the root of a PM3 clone. It will update the appropriate files in src/pocketmine/resources/vanilla, which you will need to commit.

decoder.php

<?php

/*
 *
 *  ____            _        _   __  __ _                  __  __ ____
 * |  _ \ ___   ___| | _____| |_|  \/  (_)_ __   ___      |  \/  |  _ \
 * | |_) / _ \ / __| |/ / _ \ __| |\/| | | '_ \ / _ \_____| |\/| | |_) |
 * |  __/ (_) | (__|   <  __/ |_| |  | | | | | |  __/_____| |  | |  __/
 * |_|   \___/ \___|_|\_\___|\__|_|  |_|_|_| |_|\___|     |_|  |_|_|
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * @author PocketMine Team
 * @link http://www.pocketmine.net/
 *
 *
*/

declare(strict_types=1);

namespace pocketmine\protocoltools;

use pocketmine\block\BlockFactory;
use pocketmine\entity\Attribute;
use pocketmine\item\Item;
use pocketmine\item\ItemFactory;
use pocketmine\nbt\NBT;
use pocketmine\nbt\NetworkLittleEndianNBTStream;
use pocketmine\nbt\tag\CompoundTag;
use pocketmine\nbt\tag\ListTag;
use pocketmine\network\mcpe\NetworkSession;
use pocketmine\network\mcpe\protocol\AvailableActorIdentifiersPacket;
use pocketmine\network\mcpe\protocol\BiomeDefinitionListPacket;
use pocketmine\network\mcpe\protocol\CraftingDataPacket;
use pocketmine\network\mcpe\protocol\CreativeContentPacket;
use pocketmine\network\mcpe\protocol\DataPacket;
use pocketmine\network\mcpe\protocol\PacketPool;
use pocketmine\network\mcpe\protocol\StartGamePacket;
use pocketmine\network\mcpe\protocol\types\inventory\CreativeContentEntry;
use pocketmine\utils\BinaryDataException;
use function array_chunk;
use function array_map;
use function array_values;
use function base64_decode;
use function bin2hex;
use function chr;
use function count;
use function count_chars;
use function explode;
use function file;
use function file_put_contents;
use function get_class;
use function implode;
use function is_array;
use function json_decode;
use function json_encode;
use function ksort;
use function ord;
use function random_bytes;
use function str_replace;
use function strlen;
use function var_dump;
use const JSON_PRETTY_PRINT;
use const PHP_EOL;
use const SORT_NUMERIC;
use const SORT_STRING;


$func = function(){

	global $argv;
	if(!isset($argv[1])){
		die('Usage: ' . PHP_BINARY . ' ' . __FILE__ . ' <input file>');
	}

	require 'vendor/autoload.php';

	PacketPool::init();
	Attribute::init();
	BlockFactory::init();
	ItemFactory::init();

	$handler = new class extends NetworkSession{

		public function handleDataPacket(DataPacket $packet){

		}

		public function handleStartGame(StartGamePacket $packet) : bool{
			$blockTableFile = \pocketmine\RESOURCE_PATH . '/vanilla/required_block_states.nbt';
			if(file_exists($blockTableFile)){
				echo "calculating required block table diff\n";
				$oldBlockTable = (new NetworkLittleEndianNBTStream())->read(file_get_contents($blockTableFile));
				if(!($oldBlockTable instanceof ListTag) or $oldBlockTable->getTagType() !== NBT::TAG_Compound){
					throw new \RuntimeException("unexpected block table data, expected TAG_List<TAG_Compound> root");
				}

				$newBlockTable = clone $packet->blockTable;

				$newBlockMap = [];
				/** @var CompoundTag $newState */
				foreach($newBlockTable as $newState){
					$newState->getCompoundTag("block")->removeTag("version");
					$newBlockMap[$newState->getCompoundTag("block")->getString("name")][] = $newState;
				}

				$oldBlockMap = [];
				/** @var CompoundTag $oldState */
				foreach($oldBlockTable as $oldState){
					$oldState->getCompoundTag("block")->removeTag("version");
					$oldBlockMap[$oldState->getCompoundTag("block")->getString("name")][] = $oldState;
				}
				$matched = 0;

				$newStates = [];
				foreach($newBlockMap as $id => $newStateList){
					if(!isset($oldBlockMap[$id])){
						foreach($newStateList as $state){
							$newStates[] = $state;
						}
						echo "!!! New block $id, " . count($newStateList) . " states added\n";
					}
				}
				$oldStates = [];
				foreach($oldBlockMap as $id => $oldStateList){
					if(!isset($newBlockMap[$id])){
						foreach($oldStateList as $state){
							$oldStates[] = $state;
						}

						echo "!!! Removed block $id, " . count($oldStateList) . " states removed\n";
					}
				}
				foreach($newBlockMap as $id => $newStateList){
					if(!isset($oldBlockMap[$id])){
						continue; //already checked
					}
					$oldStateList = $oldBlockMap[$id];
					do{
						$matchedThisLoop = 0;

						foreach($oldStateList as $k1 => $oldState){
							foreach($newStateList as $k2 => $newState){
								if($newState->equals($oldState)){
									unset($oldStateList[$k1]);
									unset($newStateList[$k2]);
									$matched++;
									$matchedThisLoop++;
									break 2;
								}
							}
						}
					}while($matchedThisLoop > 0);

					if(count($oldStateList) > 0){
						echo "!!! $id has removed " . count($oldStateList) . " states\n";
						foreach($oldStateList as $remaining){
							$oldStates[] = $remaining;
						}
					}
					if(count($newStateList) > 0){
						echo "!!! $id has added " . count($newStateList) . " states\n";
						foreach($newStateList as $remaining){
							$newStates[] = $remaining;
						}
					}
				}

				echo "matched $matched states\n";
				echo count($oldStates) . " old states removed: \n";
				if(count($oldStates) > 0){
					foreach($oldStates as $state){
						echo $state . "\n";
					}
				}
				echo count($newStates) . " new states added: \n";
				if(count($newStates) > 0){
					foreach($newStates as $state){
						echo $state . "\n";
					}
				}
			}
			echo "updating required blockstates table\n";
			file_put_contents($blockTableFile, (new NetworkLittleEndianNBTStream())->write($packet->blockTable));
			file_put_contents("readable_block_map.txt", str_replace("pocketmine\\nbt\\tag\\", "", (string) $packet->blockTable));

			echo "updating legacy block ID mapping table\n";
			$list = [];
			foreach($packet->blockTable as $entry){
				assert($entry instanceof CompoundTag);
				$list[$entry->getCompoundTag("block")->getString("name")] = $entry->getShort("id");
			}
			asort($list, SORT_NUMERIC);
			file_put_contents(\pocketmine\RESOURCE_PATH . '/vanilla/block_id_map.json', json_encode($list, JSON_PRETTY_PRINT));

			echo "updating legacy item ID mapping table\n";
			asort($packet->itemTable, SORT_NUMERIC);
			file_put_contents(\pocketmine\RESOURCE_PATH . '/vanilla/item_id_map.json', json_encode($packet->itemTable, JSON_PRETTY_PRINT));

			return true;
		}

		public function handleCreativeContent(CreativeContentPacket $packet) : bool{
			echo "updating creative inventory data\n";
			$items = array_map(function(CreativeContentEntry $entry) : Item{
				return $entry->getItem();
			}, $packet->getEntries());
			file_put_contents(\pocketmine\RESOURCE_PATH . 'vanilla/creativeitems.json', json_encode($items, JSON_PRETTY_PRINT));
			return true;
		}

		public function handleCraftingData(CraftingDataPacket $packet) : bool{
			echo "updating crafting data\n";
			$resultData = [];

			foreach($packet->decodedEntries as $entry){
				static $typeMap = [
					CraftingDataPacket::ENTRY_SHAPELESS => "shapeless",
					CraftingDataPacket::ENTRY_SHAPED => "shaped",
					CraftingDataPacket::ENTRY_FURNACE => "smelting",
					CraftingDataPacket::ENTRY_FURNACE_DATA => "smelting",
					CraftingDataPacket::ENTRY_MULTI => "special_hardcoded",
					CraftingDataPacket::ENTRY_SHULKER_BOX => "shapeless_shulker_box",
					CraftingDataPacket::ENTRY_SHAPELESS_CHEMISTRY => "shapeless_chemistry",
					CraftingDataPacket::ENTRY_SHAPED_CHEMISTRY => "shaped_chemistry"
				];
				if(!isset($typeMap[$entry["type"]])){
					throw new \UnexpectedValueException("Unknown recipe type ID " . $entry["type"]);
				}
				$mappedType = $typeMap[$entry["type"]];

				switch($entry["type"]){
					case CraftingDataPacket::ENTRY_SHAPED_CHEMISTRY:
					case CraftingDataPacket::ENTRY_SHAPED:
						$keys = [];

						$shape = [];
						$char = ord("A");

						$map = array_chunk($entry["input"], $entry["width"]);
						foreach($map as $x => $row){
							/**
							 * @var Item $ingredient
							 */
							foreach($row as $y => $ingredient){
								if($ingredient->getId() === 0){
									$shape[$x][$y] = " ";
								}else{
									$hash = json_encode($ingredient);
									if(isset($keys[$hash])){
										$shape[$x][$y] = $keys[$hash];
									}else{
										$keys[$hash] = $shape[$x][$y] = chr($char);
										$char++;
									}
								}
							}
						}

						unset($entry["input"]);

						$shapeEncoded = array_map(function(array $array) : string{
							return implode('', $array);
						}, $shape);
						$entry["shape"] = $shapeEncoded;
						foreach($keys as $item => $letter){
							$entry["input"][$letter] = json_decode($item, true);
						}
						unset($entry["uuid"], $entry["height"], $entry["width"], $entry["net_id"], $entry["recipe_id"]);
						break;
					case CraftingDataPacket::ENTRY_SHAPELESS:
					case CraftingDataPacket::ENTRY_SHAPELESS_CHEMISTRY:
					case CraftingDataPacket::ENTRY_SHULKER_BOX:
						unset($entry["uuid"], $entry["net_id"], $entry["recipe_id"]);

						break;
					case CraftingDataPacket::ENTRY_MULTI:
						$resultData[$mappedType][] = $entry["uuid"];
						continue 2;
					default:
						//no preprocessing needed
						break;
				}
				unset($entry["type"]);
				$resultData[$mappedType][] = $entry;
			}

			foreach($packet->potionTypeRecipes as $recipe){
				$resultData["potion_type"][] = [
					"input" => ItemFactory::get($recipe->getInputItemId(), $recipe->getInputItemMeta()),
					"output" => ItemFactory::get($recipe->getOutputItemId(), $recipe->getOutputItemMeta()),
					"ingredient" => ItemFactory::get($recipe->getIngredientItemId(), $recipe->getIngredientItemMeta())
				];
			}
			foreach($packet->potionContainerRecipes as $recipe){
				$resultData["potion_container_change"][] = [
					"input_item_id" => $recipe->getInputItemId(),
					"output_item_id" => $recipe->getOutputItemId(),
					"ingredient" => ItemFactory::get($recipe->getIngredientItemId())
				];
			}

			//this sorts the data into a canonical order to make diffs between versions reliable
			//how the data is ordered doesn't matter as long as it's reproducible
			foreach($resultData as $_type => $_recipes){
				$_sortedRecipes = [];
				foreach($_recipes as $_idx => $_recipe){
					if(is_array($_recipe)){
						ksort($_recipe, SORT_STRING);
					}
					$_key = json_encode($_recipe);
					while(isset($_sortedRecipes[$_key])){
						echo "warning: duplicated $_type recipe: $_key\n";
						$_key .= "a";
					}
					$_sortedRecipes[$_key] = $_recipe;
				}
				ksort($_sortedRecipes);
				$resultData[$_type] = array_values($_sortedRecipes);
			}

			ksort($resultData, SORT_STRING);
			foreach($resultData as $type => $entries){
				echo "$type: " . count($entries) . "\n";
			}

			file_put_contents(\pocketmine\RESOURCE_PATH . '/vanilla/recipes.json', /*str_replace("    ", "\t", */json_encode($resultData, JSON_PRETTY_PRINT)/*)*/);

			return true;
		}

		public function handleAvailableActorIdentifiers(AvailableActorIdentifiersPacket $packet) : bool{
			echo "storing actor identifiers" . PHP_EOL;
			$tag = (new NetworkLittleEndianNBTStream())->read($packet->namedtag);
			if(!($tag instanceof CompoundTag)){
				echo $tag . "\n";
				throw new \RuntimeException("unexpected actor identifiers table, expected TAG_Compound root");
			}
			if(!$tag->hasTag("idlist", ListTag::class) or $tag->getListTag("idlist")->getTagType() !== NBT::TAG_Compound){
				echo $tag . "\n";
				throw new \RuntimeException("expected TAG_List<TAG_Compound>(\"idlist\") tag inside root TAG_Compound");
			}
			if($tag->count() > 1){
				echo $tag . "\n";
				echo "!!! unexpected extra data found in available actor identifiers\n";
			}
			echo "updating legacy => string entity ID mapping table\n";
			$map = [];
			/**
			 * @var CompoundTag $thing
			 */
			foreach($tag->getListTag("idlist") as $thing){
				$map[$thing->getString("id")] = $thing->getInt("rid");
			}
			asort($map, SORT_NUMERIC);
			file_put_contents(\pocketmine\RESOURCE_PATH . '/vanilla/entity_id_map.json', json_encode($map, JSON_PRETTY_PRINT));
			echo "storing entity identifiers\n";
			file_put_contents(\pocketmine\RESOURCE_PATH . '/vanilla/entity_identifiers.nbt', $packet->namedtag);
			return true;
		}

		public function handleBiomeDefinitionList(BiomeDefinitionListPacket $packet) : bool{
			echo "storing biome definitions" . PHP_EOL;
			$defs = (new NetworkLittleEndianNBTStream())->read($packet->namedtag);
			file_put_contents(\pocketmine\RESOURCE_PATH . '/vanilla/biome_definitions.nbt', $packet->namedtag);
			return true;
		}
	};

	foreach(file($argv[1], FILE_IGNORE_NEW_LINES) as $line){
		[$type, $b64] = explode(':', $line);
		$b64 = trim($b64);
		//var_dump(base64_decode($b64));
		$pk = PacketPool::getPacket(base64_decode($b64));

		// var_dump(get_class($pk));
		try{
			$pk->decode();
		}catch(BinaryDataException $e){
			//var_dump(strlen($pk->buffer));
			//var_dump($e->getMessage());
			//var_dump(get_class($pk));
			//var_dump($type);
			//var_dump($e->getTraceAsString());
			//var_dump(strlen($pk->getBuffer()));
			continue;
		}
		$pk->handle($handler);
		//var_dump($pk->buffer);
		//var_dump(get_class($pk));
		if(!$pk->feof()){
			echo "didn't read all data from " . get_class($pk) . " (stopped at offset " . $pk->getOffset() . " of " . strlen($pk->getBuffer()) . " bytes): " . bin2hex($pk->getRemaining()) . "\n";
		}
	}

};

if(!defined('pocketmine\_PHPSTAN_ANALYSIS')){
	$func();
}

tracer.py

#!/usr/bin/python3
# -*- coding: utf-8 -*-

import frida
import sys
import json
import argparse
import subprocess
import base64
import time

def validateMode(mode):
    if mode not in 'rw':
        raise argparse.ArgumentTypeError('Unknown mode')
    return mode

parser = argparse.ArgumentParser(description='bedrock_server packet tracer')
parser.add_argument('mode', help='"r" - read, "w" - write', type=validateMode)
args = parser.parse_args()

try:
    session = frida.attach('bedrock_server')
except frida.ProcessNotFoundError:
    sys.exit('Could not find bedrock_server')
except frida.PermissionDeniedError as e:
    sys.exit(e)

logpath = './packets_' + str(time.time()) + '.txt'
logfile = open(logpath, 'wb')

def onMessage(message, data):
    if message['type'] == 'error':
        print(message['stack'])
        return
    logfile.write(str.encode(message['payload']) + b':' + base64.b64encode(data) + b'\n')

try:
    script = session.create_script("""var stringLength = new NativeFunction(Module.findExportByName(null, '_ZNKSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEE6lengthEv'), 'long', ['pointer']);

recv('input', function(message) {
    var mode = message.mode;
    var doRead = mode.includes('r');
    var doWrite = mode.includes('w');
    var count = 0;
    Module.enumerateExports('bedrock_server').forEach(function(exportedFunc) {
        if (exportedFunc.type !== 'function') {
            return;
        }
        if (!exportedFunc.name.includes('Packet')) {
            return;
        }
        if (doRead && exportedFunc.name.endsWith('Packet4readER20ReadOnlyBinaryStream')) {
            console.log("Hooking function " + exportedFunc.name);
            Interceptor.attach(exportedFunc.address, {
                onEnter: function(args) {
                        this.pointer = args[1];
                },
                onLeave: function(retval) {
                        var realAddr = Memory.readPointer(this.pointer.add(56));
                        var rlen = stringLength(realAddr);
                        send('read', Memory.readByteArray(Memory.readPointer(realAddr), rlen));
                }
            });
            count++;
        }
        if (doWrite && exportedFunc.name.endsWith('Packet5writeER12BinaryStream')) {
            console.log("Hooking function " + exportedFunc.name);
            Interceptor.attach(exportedFunc.address, {
                onEnter: function(args) {
                    this.pointer = args[1];
                },
                onLeave: function(retval) {
                    var realAddr = Memory.readPointer(this.pointer.add(56));
                    var rlen = stringLength(realAddr);
                    send('write', Memory.readByteArray(Memory.readPointer(realAddr), rlen));
                }
            });
            count++;
        }
    });
    console.log("Hooked " + count + " functions. Ready.");
});
""")

    script.on('message', onMessage)
    script.load()
    script.post({
        'type': 'input',
        'mode': args.mode
    })

    print('Logging packets to ' + logpath)
    sys.stdin.read()
except KeyboardInterrupt:
    logfile.close()
    sys.exit(0)