Talos optimizer

.gitignore

*.exe
*.ilk
*.pdb

*.ini
*.png
*.log
*.json
*.txt

.gitmodules

[submodule "ae"]
	path = ae
	url = git://github.com/CyberShadow/ae.git
[submodule "win32"]
	path = win32
	url = https://github.com/CS-svnmirror/dsource-bindings-win32

analyze.d

import std.algorithm;
import std.array;
import std.conv;
import std.exception;
import std.file;
import std.format;
import std.math;
import std.path;
import std.range;
import std.regex;
import std.stdio;
import std.string;

import ae.sys.file;
import ae.utils.aa;
import ae.utils.graphics.color;
import ae.utils.graphics.im_convert;
import ae.utils.json;
import ae.utils.regex;

enum diffVersion = 1;

void main(string[] args)
{
	enforce(args.length == 2, "Usage: analyze BASE-PROFILE");
	auto base = args[1];
	auto baseImg = parseViaIMConvert!BGR(read(buildPath("generated", base, "Screenshot.png")));

	void diff(string dir, string target)
	{
		alias COLOR = BGR;
		auto testImg = parseViaIMConvert!COLOR(read(buildPath(dir, "Screenshot.png")));
		uint diffs;
		foreach (y; 0..testImg.h)
		{
			auto baseBytes = (cast(ubyte[])baseImg.scanline(y)).ptr;
			auto testBytes = (cast(ubyte[])testImg.scanline(y)).ptr;
			foreach (i; 0..testImg.w * COLOR.sizeof)
				diffs += abs(int(*baseBytes++) - int(*testBytes++));
		}
		std.file.write(target, toJson(diffs));
	}

	static struct LogResult
	{
		float durationSeconds;
		int durationFrames;
		float averageFps, averageFpsTrimmed;
		float maxFps, minFps;
		int aiPart, physicsPart, soundPart, scenePart, shadowsPart, miscPart;
		string logText;

		@property auto metric() { return averageFpsTrimmed; }
	}

	LogResult parseLog(string dir)
	{
		auto log = cast(string)read(buildPath(dir, "Talos.log"));

		LogResult result;

		log.matchInto(regex(`.*(
\d\d:\d\d:\d\d INF:   - benchmark results -
\d\d:\d\d:\d\d INF:   
\d\d:\d\d:\d\d INF:    Duration: ([0-9\.]+) seconds \((\d+) frames\)
\d\d:\d\d:\d\d INF:     Average: ([0-9\.]+) FPS \(([0-9\.]+) w/o extremes\)
\d\d:\d\d:\d\d INF:    Extremes: ([0-9\.]+) max, ([0-9\.]+) min
\d\d:\d\d:\d\d INF:    Sections: AI=(\d+)%, physics=(\d+)%, sound=(\d+)%, scene=(\d+)%, shadows=(\d+)%, misc=(\d+)%
\d\d:\d\d:\d\d INF:       Highs: \d+ in [0-9\.]+ seconds \([0-9\.]+ FPS\)
\d\d:\d\d:\d\d INF:        Lows: \d+ in [0-9\.]+ seconds \([0-9\.]+ FPS\)
.*
)\d\d:\d\d:\d\d INF:   ``
.*`.replace("\n", "\r\n"), "ms"),
/*
\d\d:\d\d:\d\d INF:   20-30 FPS: \s*\d+%
\d\d:\d\d:\d\d INF:   30-60 FPS: \s*\d+%
\d\d:\d\d:\d\d INF:    > 60 FPS: \s*\d+%
*/
			result.logText,
			result.durationSeconds, result.durationFrames,
			result.averageFps, result.averageFpsTrimmed,
			result.maxFps, result.minFps,
			result.aiPart, result.physicsPart, result.soundPart, result.scenePart, result.shadowsPart, result.miscPart,
		).enforce("Results not found in log");
		return result;
	}

	LogResult[string] logResults;
	int[string] diffValues;

	foreach (de; chain(DirEntry("generated/" ~ base).only, dirEntries("generated", base ~ "-*", SpanMode.shallow)))
	{
	//	stderr.writeln(de.baseName);
		auto diffResult = buildPath(de, "diff.v%d.txt".format(diffVersion));
		cached!diff(de, diffResult);
		auto diffValue = diffValues[de.baseName] = readText(diffResult).to!int;
	//	writefln("%11s - %s", diffValue, de.baseName);

		auto logResult = logResults[de.baseName] = parseLog(de);
		stderr.writefln("%11s - %11s - %s", logResult.metric, readText(diffResult), de.baseName);
	}

	static struct Run
	{
		string id, value;
		LogResult result;
		bool isBase;
	}
	Run[][string] runs;

	foreach (name, logResult; logResults)
	{
		if (name == base)
			continue;
		auto varName = name.canFind('=') ? name.findSplit("=")[0].retro.findSplit("-")[0].array.retro.text : null;
		auto value = name.canFind('=') ? name.findSplit("=")[2] : null;
		runs[varName] ~= Run(name, value, logResult);
	}

	foreach (key, ref value; runs)
		value = Run(base, readText("generated/" ~ value[0].id ~ "/meta.json").jsonParse!(string[string])["defaultValue"], logResults[base], true) ~ value;

	auto texts = "texts.json".readText.jsonParse!(string[string][string]);

	static struct Result
	{
		string id, name;
		string[string] texts;
		LogResult min, max;
		int maxDiff;

		static struct Value
		{
			string id, value, valueText;
			string[string] texts, meta;
			LogResult logResult;
			int diff;
			bool isBase;
		}
		Value[] values;
	}

	runs
		.keys
		.map!(varName => Result(
			varName,
			chain(
				varName in texts ? texts[varName].keys.filter!(key => key.endsWith(" - name")).map!(key => texts[varName][key]).array : [],
				varName in texts ? texts[varName].keys.filter!(key => key.endsWith(" - brief comment")).map!(key => texts[varName][key].findSplit(". ")[0]).array : [],
				only(varName),
			).filter!(a =>  a).front,
			texts.get(varName, null),
			runs[varName].map!(run => run.result).reduce!(reduceComposite!min),
			runs[varName].map!(run => run.result).reduce!(reduceComposite!max),
			runs[varName].map!(run => diffValues[run.id]).reduce!max,
			runs[varName].map!(run => Result.Value(
				run.id,
				run.value,
				chain(
					(varName ~ "=" ~ run.value) in texts ? texts[varName ~ "=" ~ run.value].keys.filter!(key => key.endsWith(" - option value")).map!(key => texts[varName ~ "=" ~ run.value][key] ~ " (" ~ run.value ~ ")").array : [],
					only(run.value),
				).filter!(a =>  a).front,
				texts.get(varName ~ "=" ~ run.value, null),
				run.isBase ? null : readText("generated/" ~ run.id ~ "/meta.json").jsonParse!(string[string]),
				run.result,
				diffValues[run.id],
				run.isBase,
			)).array().sort!((a, b) => a.value < b.value).release(),
		))
		.array
		.toJson
		.toFile("results.json");
}

template reduceComposite(alias FUN)
{
	T reduceComposite(T)(auto ref T a, auto ref T b)
	{
		T result;
		foreach (i, v; a.tupleof)
		{
			enum n = __traits(identifier, a.tupleof[i]);
			mixin(`result.`~n~` = FUN(a.`~n~`, b.`~n~`);`);
		}
		return result;
	}
}

compare.html

<!doctype html>
<title>Compare page</title>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
<script src="http://ajax.googleapis.com/ajax/libs/jquery/2.1.3/jquery.min.js"></script>
<script src="http://misc.k3.1azy.net/tablesorter/jquery.tablesorter.min.js"></script>
<script src="http://misc.k3.1azy.net/jQuery-Before-After-Image-Comparison-Plugin-Image-Reveal/dist/jquery.imageReveal.js"></script>
<link rel="stylesheet" type="text/css" href="http://misc.k3.1azy.net/tablesorter/themes/blue/style.css"/>
<link rel="stylesheet" type="text/css" href="http://misc.k3.1azy.net/jQuery-Before-After-Image-Comparison-Plugin-Image-Reveal/dist/jquery.imageReveal.min.css"/>
<script>
	var activeRow;
	var images = {};
	$(function() {
		$.ajaxSetup({ mimeType: "text/plain" });
		$.getJSON('results.json', function(data) {
			$.each(data, function(i, row) {
				$('<tr>')
				.append($('<td>').text(row.name))
				.append($('<td>').text(row.id))
				.append($('<td>').text((row.max.averageFpsTrimmed - row.min.averageFpsTrimmed).toFixed(2)))
				.append($('<td>').text((row.maxDiff / 1000000).toFixed(2)))
				.attr('title', $.map(row.texts, function(value, id) { return value ? id + ': ' + value : ''; }).join('\n'))
				.appendTo($('#table tbody'))
				.click(function() {
					activeRow = row;

					$('.selected').removeClass('selected');
					$(this).addClass('selected');
					$('#h-active').text(row.name);

					var $selects = $('#comparisons select');
					$selects.empty();
					row.valuesByID = {};
					var setSecond = false;
					$.each(row.values, function(i, value) {
						row.valuesByID[value.id] = value;
						$option = $('<option>')
							.attr('value', value.id)
							.text(value.valueText + (value.isBase ? ' (base setting)' : ''));
						$selects.each(function() { $option.clone().appendTo($(this)); });
						if (value.isBase) {
							$('#comparison-1 option:last-child').prop('selected', true);
						}
						else
						if (!setSecond) {
							$('#comparison-2 option:last-child').prop('selected', true);
							setSecond = true;
						}
					});
					$selects.change();
				})
			});
			$('#table').tablesorter();
		});
		$('#comparison-1 > *').clone().appendTo($('#comparison-2'));
		$('#comparison-2 h3').text('Comparison - Right');
		$('#comparisons select').change(function() {
			var id = $(this).val();
			var value = activeRow.valuesByID[id];
			var $desc = $(this).closest('td').find('.description');
			var n = $(this).closest('td').attr('id').split('-')[1];
			$desc.empty();
			$('<div>')
				.append(
					$('<a>')
					.attr('href', 'generated/' + value.id + '/Screenshot.png')
					.text('Full screenshot')
				,
					' &middot; '
				,
					$('<a>')
					.attr('href', 'generated/' + value.id + '/Talos.ini')
					.text('INI file')
				,
					' &middot; '
				,
					$('<a>')
					.attr('href', 'generated/' + value.id + '/Talos.log')
					.text('Log file')
				)
				.appendTo($desc)
			;
			$('<div>')
				.text('INI line: ')
				.append($('<tt>')
					.text(activeRow.id + ' = ' + (value.isBase ? activeRow.values[1].meta.rawDefaultValue : value.meta.rawValue))
				).appendTo($desc)
			;
			$.each(value.texts, function(id, value) {
				if (value)
					$desc.append($('<div>').text(id + ': ' + value));
			});

			$('#demo').remove();
			images[n] = 'generated/' + value.id + '/Screenshot.png';
			if (images['1'] && images['2']) {
				$('body').append(
					$('<div>')
					.attr('id', 'demo')
					.append($('<img>').attr('src', images['1']))
					.append($('<img>').attr('src', images['2']))
				);
				var loadCounter = 0;
				$('#demo img').load(function() {
					loadCounter++;
					console.log(loadCounter);
					if (loadCounter == 2) {
						$('#demo').imageReveal({
							barWidth: 15,
							touchBarWidth: 40,
							paddingLeft: 0,
							paddingRight: 0,
							startPosition: 0.25,
							showCaption: true,
							captionChange: 0.5,
							width: $('#demo img').width(),
							height: $('#demo img').height()
						});
					}
				});
			}
		});
	});
</script>

<style>
#table-container {
	overflow-y: auto;
	height: 19em;
	border: 1px solid #aaa;
	margin-bottom: 2em;
}
#table {
	margin: 0;
}
#table td {
	cursor: pointer;
}
.selected td {
	background-color: #eeeeff !important;
}

#comparisons {
	width: 100%;
}
#comparisons td {
	vertical-align: top;
	width: 50%;
}
#comparisons select {
	width: 100%;
}
#comparisons .description {
	height: 10em;
	overflow-y: auto;
}
h3 {
	margin: 0;
	text-align: center;
}
#demo {
	width: 100%;
}
#demo img {
	max-width: 100%;
}
</style>

<div id="table-container">
	<table id="table" class="tablesorter">
		<thead>
			<tr>
				<th>Name</th>
				<th>ID</th>
				<th>FPS variation</th>
				<th>Visual impact</th>
			</tr>
		</thead>
		<tbody>
		</tbody>
	</table>
</div>

<h3 id="h-active"></h3>
<table id="comparisons">
	<tr>
		<td id="comparison-1">
			<h3>Comparison - Left</h3>
			<select></select>
			<div class="description"></div>
		</td>
		<td id="comparison-2">
			<!-- will be cloned in JS -->
		</td>
	</tr>
</table>

gen.d

/*
	Generate a number of profiles by varying settings from the
	possible values collected from a directory of source profiles.

	Usage:
	1. Create "presets" directory with a bunch of .ini files.
	   For example, start the game, select some presets, exit the game,
	   copy over and rename the .ini file, repeat.
	2. Run this program. A "generated" directory will be created, with
	   many subdirectories, each containing a "Talos.ini". You can feed
	   this directory to run.d,
*/

import std.algorithm;
import std.array;
import std.file;
import std.path;
import std.stdio;
import std.string;

import ae.sys.file;
import ae.utils.json;

void main(string[] base)
{
	string[][string] settings;
	foreach (fn; dirEntries("presets", "*.ini", SpanMode.shallow))
	{
		foreach (line; fn.readText().splitLines())
			if (line.length)
			{
				auto s = line.findSplit(" = ");
				if (s[0] == "prj_strLastAutoDetectSetup")
					continue; // too long
				if (s[0] !in settings)
					settings[s[0]] = [s[2]];
				else
				if (settings[s[0]].countUntil(s[2]) < 0)
					settings[s[0]] ~= s[2];
			}
	}

	foreach (name; settings.keys.sort())
		if (settings[name].length > 1)
			writeln(name, ":", settings[name]);

	foreach (fn; dirEntries("presets", "*.ini", SpanMode.shallow))
	{
		auto lines = fn.readText().splitLines();

		void saveProfile(string name, string value, string defaultValue)
		{
			auto iniFile = "generated/" ~ name ~ "/Talos.ini";
			ensurePathExists(iniFile);
			std.file.write(iniFile, lines.join("\r\n"));

			struct Meta
			{
				string value, rawValue;
				string defaultValue, rawDefaultValue;
			}
			if (value && defaultValue)
				Meta(value.sanitizeValue(), value, defaultValue.sanitizeValue(), defaultValue).toJson.toFile("generated/" ~ name ~ "/meta.json");
		}

		auto profileName = fn.baseName.stripExtension();
		saveProfile(profileName, null, null);

		foreach (i, line; lines)
			if (line.length)
			{
				auto s = line.findSplit(" = ");
				auto defaultValue = s[2];
				if (s[0] in settings && settings[s[0]].length > 1)
				{
					foreach (value; settings[s[0]])
						if (value != s[2])
						{
							lines[i] = s[0] ~ " = " ~ value;
							saveProfile(profileName ~ "-" ~ s[0] ~ "=" ~ sanitizeValue(value), value, defaultValue);
						}
					lines[i] = line;
				}
			}
	}
}

string sanitizeValue(string value)
{
	return value
		.split(`;`)[0]
		.replace(`"`, ``)
	;
}

gettexts.d

enum gameDir = `C:\Program Files (x86)\Steam\steamapps\common\The Talos Principle\`;

import std.algorithm;
import std.file;
import std.path;
import std.stdio;
import std.typecons;
import std.zip;

import ae.sys.datamm;
import ae.sys.file;
import ae.utils.json;
import ae.utils.xmllite;

void main()
{
	string[string][string] texts;

	auto zipData = mapFile(gameDir ~ `Content\Talos\All_01.gro`, MmMode.read);
	auto archive = scoped!ZipArchive(zipData.mcontents);
	foreach (name, entry; archive.directory)
		if (name.endsWith("_cvars.xml"))
		{
			auto xml = cast(string)archive.expand(entry);
			foreach (cvar; xml.xmlParse["HELP"].findChildren("CVARS").map!(child => child.children).joiner)
			{
				texts[cvar["NAME"].text][name.baseName.stripExtension ~ " - brief comment"] = cvar["BRIEF_COMMENT"].text;
				texts[cvar["NAME"].text][name.baseName.stripExtension ~ " - detailed comment"] = cvar["DETAIL_COMMENT"].text;
			}
	}

	//string[string][string] locIDs;

	void doLoc(string var, string name, string text)
	{
		auto p = text.findSplit("=");
		texts[var][name ~ " (localization ID)"] = p[0];
		//locIDs[var][name] = p[0];
		//texts[var][name ~ " (untranslated)"] = p[2];
		texts[var][name] = p[2];
	}

	foreach (de; dirEntries(gameDir ~ `Content\Talos\Config`, "*.xml", SpanMode.shallow))
	{
		auto xml = readText(de);
		foreach (item; xml.xmlParse["menu"].findChildren("item"))
		{
			doLoc(item.attributes["cvar"], de.baseName.stripExtension ~ " - name", item.attributes["name"]);
			doLoc(item.attributes["cvar"], de.baseName.stripExtension ~ " - tooltip", item.attributes["tooltip"]);
			foreach (widget; item.findChildren("widget"))
				foreach (choice; widget.findChildren("choice"))
					doLoc(item.attributes["cvar"] ~ "=" ~ choice.attributes["value"], de.baseName.stripExtension ~ " - option value", choice.attributes["name"]);
		}
	}

	texts.toJson.toFile("texts.json");
}

linkscreens.d

import std.file;
import std.path;

import ae.sys.file;

void main()
{
	foreach (de; "generated".dirEntries(SpanMode.shallow))
	{
		auto scr = buildPath(de, "Screenshot.png");
		if (scr.exists)
		{
			auto dst = buildPath("screenshots", de.baseName ~ scr.extension);
			if (!dst.exists)
			{
				ensurePathExists(dst);
				hardLink(scr, dst);
			}
		}
	}
}

run.d

/*
	Usage:
	1. Start Steam in offline mode
	2. Back up your game data ("C:\Program Files (x86)\Steam\userdata\<UID>\257510\")
	3. Adjust settings below
	4. Create some profile batches to run, e.g. using gen.d
	   (Directory structure: batches/<batch-name>/<profile-dir>/Talos.ini)
	5. Run this program
*/

enum duration = 30; // how many seconds to benchmark
enum gameDir = `C:\Program Files (x86)\Steam\steamapps\common\The Talos Principle\`;
enum dataDir = `C:\Program Files (x86)\Steam\userdata\25827405\257510\`;
enum delay = 500; // key press delay - increase if bot gets stuck while entering console commands

import std.algorithm;
import std.array;
import std.conv;
import std.exception;
import std.file;
import std.path;
import std.process;
import std.stdio;
import std.string;
import std.utf;

import win32.winbase;
import win32.winnt;
import win32.winuser;

import ae.sys.clipboard;
import ae.sys.file;
import ae.sys.windows.exception;
import ae.sys.windows.input;

void toggleConsole()
{
	press(192);
	Sleep(delay);
}

void sendCommand(string command)
{
	setClipboardText(command);
	Sleep(delay);

	keyDown(VK_CONTROL);
	Sleep(delay);
	press('K');
	Sleep(delay);
	keyUp(VK_CONTROL);
	press(VK_RETURN);
}

string[] commands = `
`.strip().splitLines();

enum logFile = gameDir ~ `Log\Talos.log`;
enum exeFile = gameDir ~ `Bin\Talos.exe`;

void killGame()
{
	killAll("steam.exe");
	killAll("steamwebhelper.exe");
	killAll("talos.exe");
}

void cleanup()
{
	killGame();
	Sleep(100);

	if (logFile.exists)
		logFile[].remove();

	auto runFile = gameDir ~ `Temp\run.txt`;
	if (runFile.exists)
		runFile.remove();
}

void patchSteam()
{
	auto loginFile = environment[`ProgramFiles(x86)`] ~ `\Steam\config\loginusers.vdf`;
	loginFile
		.readText()
		.replace(`"WantsOfflineMode"		"0"`, `"WantsOfflineMode"		"1"`)
		.replace(`"SkipOfflineModeWarning"		"0"`, `"SkipOfflineModeWarning"		"1"`)
		.toFile(loginFile)
	;
}

void startGame()
{
	spawnProcess(exeFile);
	patchSteam();
}

void waitForLog(string needle)
{
	while (!logFile.exists)
		Sleep(100);

	while (readShared(logFile).indexOf(needle) < 0)
		Sleep(10);
}

void benchmarkProfile(string profileDir)
{
	auto profileFile = buildPath(profileDir, "Talos.ini");
    writeln("Benchmarking profile: ", profileFile);
	copy(profileFile, dataDir ~ `local\Talos.ini`);

	cleanup();
	startGame();

	waitForLog(`Started simulation on 'Content/Talos/Levels/Menu/Intro.wld'`);
	Sleep(1000);
	press(VK_ESCAPE); // skip intro
	Sleep(1000);
	press(VK_ESCAPE); // enter menu (and load profile)

	waitForLog(`Stopping world 'Content/Talos/Levels/Menu/Intro.wld'.`);
	Sleep(1000);
	toggleConsole();
	sendCommand(`gfx_iScreenShotFormat=4`);
	sendCommand(`prjStartNewTalosGame("Content/Talos/Levels/Demo.nfo")`);
	toggleConsole();

	waitForLog(`Started simulation on 'Content/Talos/Levels/Demo.wld'`);
	Sleep(10_000);
	press(VK_F11);
	waitForLog(`Screenshot taken`);
	Sleep(2_500);

	toggleConsole();
	sendCommand(`cht_bEnableCheats=2`);
	sendCommand(`bot_bSkipTerminalsAndMessages=1`);
	sendCommand(`cht_bAutoTestBot=1`);
	sendCommand(`bmkStartBenchmarking(5, ` ~ text(duration) ~ `)`);
	toggleConsole();
	//waitForLog(`Auto test bot started on world: Content/Talos/Levels/Demo.wld`);

	waitForLog(`- benchmark results -`);
	Sleep(1000);
	killGame();

	copy(logFile, buildPath(profileDir, "Talos.log"));
	auto screenshot = (gameDir ~ `Temp\ScreenShots\`)
		.dirEntries("Demo_*.png", SpanMode.shallow)
		.array
		.sort!((a, b) => a.timeLastModified > b.timeLastModified)
		.front
		.name;
	copy(screenshot, buildPath(profileDir, "Screenshot.png"));
	remove(screenshot);
}

void main()
{
	foreach (batch; dirEntries("batches", SpanMode.shallow))
		foreach (de; dirEntries(batch.name, SpanMode.shallow))
			if (!exists(de.buildPath("Talos.log"))
			 || !exists(de.buildPath("Screenshot.png")))
				benchmarkProfile(de.name);
}

string readShared(string fn)
{
	try
	{
		auto h = CreateFileW(toUTF16z(fn), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, null, OPEN_EXISTING, 0, HANDLE.init);
		wenforce(h != INVALID_HANDLE_VALUE);
		File f;
		f.windowsHandleOpen(h, "rb");
		auto result = new char[cast(size_t)f.size];
		if (result.length == 0)
			return null;
		f.rawRead(result);
		return result.assumeUnique();
	}
	catch
		return null;
}

void killAll(string exe)
{
	spawnProcess(["taskkill", "/F", "/IM", exe]).wait();
}