Pak & Unpack system

README.md

This is a bare-bones PAK / UNPAK system. Big thanks to @hasufel, he wrote all the original code, I just cleaned it up.

1. If you compile PakPacker as an openfl project using "lime test neko -clean" it will package all your text/image assets in the /assets folder into:

   • resources.pak
   • header

2. To use PakLibrary.hx:

   • Put resources.pak & header into your assets/ folder
   • instantiate like so: var paklib = new PakLibrary() (or pass in asset id's for header & resources explicitly, otherwise uses defaults)
   • get bitmapData: paklib.getBitmapData("brandlogo.png");
   • get text: paklib.getText("brandlogo.xml");

3. PakPacker can eventually be rolled into Openfl/Lime tools, PakLibrary can eventually be rolled into OpenFL/Lime assets.

PakLibrary.hx

package;

import haxe.io.Bytes;
import lime.utils.UInt8Array;
import lime.graphics.Image;
import lime.graphics.ImageBuffer;
import openfl.Assets.AssetLibrary;
import openfl.Assets;
import openfl.Lib;
import openfl.utils.ByteArray;

/**
 * Loads images and text files from one big LZMA-compressed binary blob.
 * Assets load much faster than reading from disk using regular Assets.get(), but the entire contents of the PAK will stay in ram from the moment you create it
 * @author hasufel, larsiusprime
 */
class PakLibrary extends AssetLibrary
{
	private var _images:Map<String, Image>;
	private var _strings:Map<String, String>;
	
	public function new(header:String = "assets/header", pak:String = "assets/resources.pak") {
		
		super();
		init(header, pak);
		
	}
	
	override public function unload():Void {
		
		for (key in _images.keys())
		{
			_images.remove(key);
		}
		
		for (key in _strings.keys())
		{
			_strings.remove(key);
		}
		
		_strings = null;
		
	}
	
	override public function exists (id:String, type:String):Bool {
		
		var b = false;
		
		if (type == cast AssetType.IMAGE) {
			
			b = _images != null && _images.exists(id);
			
		}
		else if (type == cast AssetType.TEXT) {
			
			b = _strings != null && _strings.exists(id);
			
		}
		else if (type == cast AssetType.BINARY) {
			
			b = (_images != null && _images.exists(id)) || (_strings != null && _strings.exists(id));
			
		}
		
		return b;
		
	}
	
	override public function getImage(id:String):Image {
		
		if (_images == null) return null;
		return _images.get(id);
		
	}
	
	override public function getText(id:String):String {
		
		if (_strings == null) return null;
		return _strings.get(id);
		
	}
	
	/**********PRIVATE**********/
	
	//These temp structures are used to set up all the data and get it into ram, and are then destroyed

	private static inline var PAK_ASSET_TEXT:Int = 0;
	private static inline var PAK_ASSET_IMAGE:Int = 1;
	
	private var _tempHeaderBytes:Bytes;
	private var _tempAssetBytes:Bytes;
	private var _tempHeaderDetails:Array<Array<Int>>;
	private var _tempHeaderFilenames:Array<String>;
	
	private var _headerSource:String;
	private var _pakSource:String;
	
	private function init(header:String, pak:String):Void {
		
		_pakSource = pak;
		_headerSource = header;
		_tempHeaderBytes = Assets.getBytes(header);
		_tempAssetBytes = Assets.getBytes(pak);
		
		var fail = false;
		
		if (_tempHeaderBytes == null) {
			
			trace("[PakLibrary] Header asset \"" + header + "\" does not exist");
			fail = true;
			
		}
		if (_tempAssetBytes == null) {
			
			trace("[PakLibrary] Resource asset \"" + pak + "\" does not exist");
			fail = true;
			
		}
		
		if (!fail) {
			
			makeHeaderDetails();
			loadAssetsFromPakIntoRam();
			
		}
	}
	
	private function makeHeaderDetails():Void {
		
		//Detail entry formats are:
		// Image: [typeFlag, numBytes, width, height]
		// Text:  [typeFlag, numBytes]
		
		//Create two temporary data structures to hold our data:
		_tempHeaderDetails = [];
		_tempHeaderFilenames = [];
		
		//Get the bytes from the header and uncompress them:
		var ba:ByteArray = ByteArray.fromBytes(_tempHeaderBytes);
		ba.uncompress(LZMA);
		
		var howMany:Int = ba.readInt();
		
		//March through the bytes one by one
		for (i in 0...howMany) {
			
			var numBytes:Int = ba.readInt();		//Byte 0: number of bytes to read from _assetBytes
			var typeFlag:Int = ba.readInt();		//Byte 1: type flag
			
			var details:Array<Int> = [typeFlag, numBytes];
			
			if (typeFlag == PAK_ASSET_IMAGE) {	//if it's an image:
				
				var w:Int = ba.readInt();			//Byte 2: width of image
				var h:Int = ba.readInt();			//Byte 3: height of image
				details.push(w);
				details.push(h);
				
			}
			
			var filenameSize:Int = ba.readShort();						//Next Byte: size of filename
			var filenameText:String = ba.readUTFBytes(filenameSize);	//Read that many bytes as the filename
			
			//make sure it's a  valid file type
			if (typeFlag == PAK_ASSET_IMAGE || typeFlag == PAK_ASSET_TEXT) {
				
				_tempHeaderDetails.push(details);
				_tempHeaderFilenames.push(filenameText);
			}
		}
	}
	
	private function loadAssetsFromPakIntoRam():Void { 
		
		//hasufel notes: this could be ported in multithreading. limit to a pool of 4 rolling threads (to match with cores of platforms)
		
		_images = new Map<String, Image>();
		_strings = new Map<String, String>();
		
		for (i in 0..._tempHeaderDetails.length) {
			
			var details  = _tempHeaderDetails[i];
			var filename = _tempHeaderFilenames[i];
			
			//load the images and text files and store them in the permanent easy-access map structures
			
			switch(details[0]) {
				
				case PAK_ASSET_TEXT : _strings.set(filename, getTempPakText(i, details));
				case PAK_ASSET_IMAGE: _images.set(filename, getTempPakImage(i, details));
				
			}
		}
		
		//nullify temporary structures and assets from ram, we dont need references anymore
		
		_tempAssetBytes = null;
		_tempHeaderBytes = null;
		_tempHeaderFilenames = null;
		
		for (i in 0..._tempHeaderDetails.length) {
			
			_tempHeaderDetails[i] = null;
			
		}
		_tempHeaderDetails = null;
		
		//clear the cache of the bytes we loaded to start this whole operation
		Assets.cache.clear(_headerSource);
		Assets.cache.clear(_pakSource);
	}
	
	private function getTempPakText(n:Int, details:Array<Int>):String {
		
		if (details == null) return null;
		if (details[0] != PAK_ASSET_TEXT) return null;
		
		var bytes = getTempPakFileBytes(n);
		var byteArray = deflateBytes(bytes);
		
		var str:String = null;
		
		if (byteArray != null) {
			
			var s:String = byteArray.toString();
			str = s;
			
		}
		
		bytes = null;
		byteArray = null;
		
		return str;
	}
	
	private function getTempPakImage(n:Int, details:Array<Int>):Image {
		
		if (details == null) return null;
		if (details[0] != PAK_ASSET_IMAGE) return null;
		
		var bytes = getTempPakFileBytes(n);
		var byteArray = deflateBytes(bytes);
		
		var img:Image = null;
		
		if (byteArray != null) {
			
			//png, get dimensions:
			var w:Int = details[2];
			var h:Int = details[3];
			
			var buffer = new ImageBuffer (new UInt8Array (w * h * 4), w, h);
			buffer.format = BGRA32;
			buffer.premultiplied = true;
			
			img = new Image (buffer, 0, 0, w, h);
			img.setPixels(img.rect, byteArray, ARGB32);
		}
		
		bytes = null;
		byteArray = null;
		
		return img;
	}
	
	private function getTempPakFileBytes(n:Int):Bytes {
		
		if (n < 0 || _tempHeaderDetails.length <= n) return null;
		
		var start:Int = 0;
		
		//get to the correct starting byte offset
		for (i in 0...n) start += _tempHeaderDetails[i][1];
		
		//allocate the correct number of bytes
		var b:Bytes = Bytes.alloc(_tempHeaderDetails[n][1]);
		
		//fill the bytes from the pak file
		b.blit(0, _tempAssetBytes, start, _tempHeaderDetails[n][1]);
		
		return b;
		
	}
	
	private inline function deflateBytes(b:Bytes):ByteArray {
		
		//decompress the byte array
		var d:ByteArray = ByteArray.fromBytes(b);
		d.uncompress(LZMA);
		d.position = 0;
		return d;
		
	}
}

PakPacker.hx

package;

/* Hasufel 2015 */
/* Packing system */
/* Will grab all files from "assets/" directory */
/* then pack them in the bin assets/ folder for now */
/* resulting in resources.pak and header being the key to know */
/* where and how the files are packed */
/* You can encrypt header for instance to prevent unpaking */
/* Path might change to userdirectory or desktop */

#if !lime_legacy
import lime.system.System;
#else
import openfl.utils.SystemPath;
#end
import openfl.Assets;

import flash.display.Bitmap;
import flash.display.BitmapData;
import flash.display.Sprite;
import flash.geom.Rectangle;
import flash.Lib;
import flash.utils.ByteArray;

import sys.io.File;
import sys.io.FileSeek;
import sys.FileSystem;
import haxe.io.Bytes;

class PakSystem {

	public function new() {
		haxe.Timer.delay(init,50);
	}

	private function init():Void {
		pakDaFiles("assets/thing1","thing1");
		pakDaFiles("assets/thing2","thing2");
		pakDaFiles("assets/thing3","thing3");
	}

	private function recursivePush(files:Array<String>, path:String, filename:String)
	{
		if (FileSystem.isDirectory(path+filename))
		{
			path = path + filename+"/";
			var fil = FileSystem.readDirectory(path);
			for (ff in fil)
			{
				recursivePush(files, path, ff);
			}
		}
		else
		{
			trace(path + filename);
			files.push(path+filename);
		}
	}
	
	private function pakDaFiles(subDir:String,outName:String):Void {
		
		var ba:Array<ByteArray> = new Array<ByteArray>();
		var path:String = '';
		
		#if !lime_legacy
		path = System.applicationDirectory;
		#else
		path = SystemPath.applicationDirectory;
		#end
		
		trace('current path:' + path);
		
		var assetsPath:String = "assets/" + subDir;
		
		var fil = FileSystem.readDirectory(assetsPath);
		
		var files:Array<String> = [];
		
		for(ff in fil) {
			recursivePush(files, assetsPath+"/", ff);
		}
		
		var textExtensions  = ['xml', 'txt', 'csv', 'tsv'];
		var legalExtensions = textExtensions.concat(["png"]);
		
		var header:ByteArray = new ByteArray();
		
		var pruneFiles = [];
		
		for (i in 0...files.length) {
			for (extension in legalExtensions)
			{
				if (files[i].lastIndexOf("." + extension) == files[i].length - 4)
				{
					pruneFiles.push(files[i]);
				}
			}
		}
		
		files = null;
		files = pruneFiles;
		
		header.writeInt(files.length);
		
		var j:Int = 0;
		
		for (i in 0...files.length) {
			
			trace('packing file ' + i + '(' + files[i] + ')' + '...');
			var ext:String = files[i].split('.')[1];
			
			if (textExtensions.indexOf(ext) != -1) {
				
				var c = File.getBytes(files[i]);
				ba[j] = new ByteArray();
				ba[j].writeBytes(c);
				ba[j].compress(LZMA);
				header.writeInt(ba[j].length);		//NUM_BYTES
				header.writeInt(0);					//TYPE FLAG (0 means text)
				header.writeShort(files[i].length);	//SIZE OF FILENAME
				header.writeUTFBytes(files[i]);		//FILENAME
				j++;
			}
			
			else if (ext == 'png') {
				
				//get dims
				var bmd:BitmapData = Assets.getBitmapData(files[i]);
				
				ba[j] = new ByteArray();
				
				ba[j].writeBytes(bmd.getPixels(bmd.rect));
				ba[j].compress(LZMA);
				
				header.writeInt(ba[j].length);		//NUM_BYTES
				header.writeInt(1);					//TYPE FLAG (1 means PNG)
				header.writeInt(bmd.width);			//WIDTH
				header.writeInt(bmd.height);		//HEIGHT
				header.writeShort(files[i].length);	//SIZE OF FILENAME
				header.writeUTFBytes(files[i]);		//FILENAME
				bmd.dispose();
				bmd = null;
				j++;
			}
		}
		
		trace('packing finished, saving on disk...');
		
		var f = File.write('assets/'+outName+'.pak', true);
		var count:Int = 0;
		for (i in 0...ba.length)
		{
			f.write(ba[i]);
			count += ba[i].length;
		}
		f.flush();
		f.close();
		trace('saved packed files, writing header...');
		header.compress(LZMA);//LZMA
		
		var g = File.write('assets/'+outName+'.header', true);
		g.write(header);
		g.flush();
		g.close();
		
		trace('header wrote, all finished.');
	}

	static public function main() {
		new PakSystem();
	}
}

Hi Lars,
At the moment I'm converting flash game to Openfl and in the game have more than 3000 small images , each as separate file. In HTML5 target it's take too much time ( > 5 min, ) to embeded ( cache in the browser ) all images, so I have some options.

1. To load image asynchronously ( in first call) , but then need to change some of the logic of the game
2. Load as lzip package ( or other pack system, as you show in this gist)
3. Create texture atlases and load them as embeded.
   What is your opinion ? What is the best way to load > 3000 images for HTML5 ?
   Thank you in advance.