Crosstable Translator Tool

index.js

const fs = require("fs");
const path = require("path");
const stringSimilarity = require("string-similarity"); // You need to npm install these modules
const matchAll = require("string.prototype.matchall");
const Confirm = require("prompt-confirm");
const coloredText = require("ansi-cyan");
const greenText = require("ansi-bold");

const nativesH = fs.readFileSync("natives.log").toString().replace(/\r/g, ""); // This must be a list of native hashes
const nativesHeaderFile = fs.readFileSync("natives.h").toString().replace(/\r/g, ""); // This must be a generic natives.h file
const crossTableFile = "crosstable.txt";
const errorHashesFile = "errorhashes.txt";
const oldScriptsFolder = "A:\\GTAV-1.46-Scripts-master"; // Change this!
const newScriptsFolder = "A:\\Saved 1.47 Scripts";
const searchRadius = 500;
const SimilarityConfirm = new Confirm("Is the native equal (equal arguments, etc.)?");

var alreadyNatives = [];
var natives = [];
var oldScriptFiles = [];
var newScriptFiles = [];
var nativeNamespaces = [];

if(fs.existsSync(crossTableFile)) {
	for (let crossLine of fs.readFileSync(crossTableFile).toString().replace(/\r/g, "").split("\n")) {
		if(crossLine.includes("},"))
			alreadyNatives.push(crossLine.split(", ")[1].split("}")[0]);
	}

	if(fs.existsSync(errorHashesFile))
		for (let crossLine of fs.readFileSync(errorHashesFile).toString().replace(/\r/g, "").split("\n")) {
			if(crossLine.includes("0x"))
				alreadyNatives.push(crossLine.trim());
		}
}

nativesH.split("\n").forEach(line => {
	if (!line.includes("[") && line.includes("0x") && !alreadyNatives.includes(line.trim())) {
		natives.push(line.trim());
	}
});
console.log("Loaded natives: " + natives.length);

fs.readdirSync(oldScriptsFolder).forEach(file => {
	file = path.join(oldScriptsFolder, file);
	oldScriptFiles.push({
		fileSize: fs.statSync(file).size,
		file: file
	});
});
fs.readdirSync(newScriptsFolder).forEach(file => {
	file = path.join(newScriptsFolder, file);
	newScriptFiles.push({
		fileSize: fs.statSync(file).size,
		file: file
	});
});
oldScriptFiles = oldScriptFiles.sort((a, b) => {
	return a.fileSize - b.fileSize;
});
newScriptFiles = newScriptFiles.sort((a, b) => {
	return a.fileSize - b.fileSize;
});
console.log("Loaded scripts: " + oldScriptFiles.length + "/" + newScriptFiles.length);

const nativeHeaderRegexName = / ([a-zA-Z0-9_]+)\(/g;
const nativeHeaderRegexHash = />\((0x[A-Z0-9]+)/g;
var currentNamespace = "";
var currentNamespaceNatives = [];
nativesHeaderFile.split("\n").forEach(line => {
	if (line.includes("namespace ")) {
		if (currentNamespaceNatives.length > 0) {
			nativeNamespaces.push({
				namespace: currentNamespace,
				natives: currentNamespaceNatives
			});
		}
		currentNamespace = line.split(" ")[1];
	} else if (line.includes(">(0x")) {
		let nativeMatchName = matchAll(line, nativeHeaderRegexName);
		let nativeMatchHash = matchAll(line, nativeHeaderRegexHash);

		for (let matchNameMatch of nativeMatchName) {
			for (let matchHashMatch of nativeMatchHash) {
				currentNamespaceNatives.push({
					name: matchNameMatch[1],
					hash: matchHashMatch[1]
				});
				break;
			}
			break;
		}

	}
});
nativeNamespaces.push({
	namespace: currentNamespace,
	natives: currentNamespaceNatives
});
console.log("Native Namespaces: " + nativeNamespaces.length);

function searchNativeInFiles(native) {
	for (let oldFile of oldScriptFiles) {
		let oldFileText = fs.readFileSync(oldFile.file).toString();

		for (let newFile of newScriptFiles) {
			if (newFile.file.endsWith(path.basename(oldFile.file))) {
				let newFileText = fs.readFileSync(newFile.file).toString();

				if (newFileText.includes(native + "(")) {
					console.log("Found native " + native + " in " + path.basename(oldFile.file));
					return [oldFileText, newFileText];
				}
			}
		}
	}
	return false;
}



// Group 1 = Function/Array Name; Group 2 = Numbers in comments; Group 3 = Global f_vars numbers; Group 4 = Global Numbers; Group 5 = Local Vars Numbers
const changingStuffRegex = /(?:([a-zA-Z_][_a-zA-Z0-9]+?)[([])|(?:\/\*(\d+)\*\/)|(?:\.f_(\d+))|(?:Global_(\d+))|(?:Local_(\d+))/g;

var changedOldLines = [];

function clearLines(textFileLines) {
	changedOldLines = [];
	for(let i = 0; i < textFileLines.length; i++) {
		let line = textFileLines[i];
		if (line === undefined)
			continue;
		let changingStuffMatches = matchAll(line, changingStuffRegex);

		for (let match of changingStuffMatches) {
			if (match[1] !== undefined) { // Function/Array
				line = line.replace(match[0], match[0].replace(match[1], ""));
				continue;
			}
			if (match[2] !== undefined) { // Comment Number
				line = line.replace(match[0], match[0].replace(match[2], ""));
				continue;
			}
			if (match[3] !== undefined) { // Global f_var number
				line = line.replace(match[0], match[0].replace(match[3], ""));
				continue;
			}
			if (match[4] !== undefined) { // Global number
				line = line.replace(match[0], match[0].replace(match[4], ""));
				continue;
			}
			if (match[5] !== undefined) { // Local Var Number
				line = line.replace(match[0], match[0].replace(match[5], ""));
				continue;
			}
		}

		changedOldLines.push(line);
	}
}

function getNearbyLines(textFileLines, lineNumber) {
	let nearbyLines = [];

	for (let i = Math.max(Math.round(lineNumber / 2), lineNumber - searchRadius); i < Math.min(Math.round(lineNumber * 2), lineNumber + searchRadius); i++) {
		let line = textFileLines[i];
		if (line === undefined)
			continue;
		let lineDistance = lineNumber - i;

		nearbyLines.push(changedOldLines[i] + lineDistance);
	}

	return nearbyLines;
}

async function checkNative(native) {
	let nativeFiles = searchNativeInFiles(native);
	if (nativeFiles != false) {
		console.log("Analyzing native " + native);
		let oldFileText = nativeFiles[0];
		let newFileText = nativeFiles[1];
		let oldFileTextLines = oldFileText.replace(/\r/g, "").split("\n");
		let newFileTextLines = newFileText.replace(/\r/g, "").split("\n");

		let newLinesCount = 0;
		let analyzed = false;

		let nativeResult = undefined;
		for (let line of newFileTextLines) {
			if (line.includes("unk_" + native) && !analyzed) {
				clearLines(newFileTextLines);
				let newNearbyLines = getNearbyLines(newFileTextLines, newLinesCount);
				analyzed = true;
				//console.log(newNearbyLines);
				let newNearbyLinesTxt = "";
				newNearbyLines.forEach(nearbyLineN => {
					newNearbyLinesTxt += nearbyLineN;
				});

				clearLines(oldFileTextLines);
				let totalLines = [];
				for (let x = Math.max(Math.round(newLinesCount / 2), 0); x < Math.min(oldFileTextLines.length, Math.min(Math.round(newLinesCount * 2), Infinity)); x++) {
					let oldLLine = oldFileTextLines[x];

					if (!oldLLine.includes("(") || !(oldLLine.includes("::") || oldLLine.includes("unk_"))) {
						continue;
					}

					let oldNearbyLines = getNearbyLines(oldFileTextLines, x);
					let oldNearbyLinesTxt = "";
					oldNearbyLines.forEach(nearbyLineO => {
						oldNearbyLinesTxt += nearbyLineO;
					});

					let nearbySimilarity = stringSimilarity.compareTwoStrings(newNearbyLinesTxt, oldNearbyLinesTxt);
					totalLines.push({
						similarity: nearbySimilarity,
						line: x
					});
				}

				totalLines.sort((a, b) => {
					return b.similarity - a.similarity;
				});

				for (let z = 0; z < Math.min(5, totalLines.length); z++) {
					let totalLine = totalLines[z];

					if (totalLine.similarity >= 0.9) { // Auto Similarity threshold
						nativeResult = {
							oldLine: totalLine.line,
							newLine: newLinesCount,
							oldLineTxt: oldFileTextLines[totalLine.line],
							newLineTxt: line
						};
						break;
					} else {
						process.stdout.write("\x07");
						console.log("");
						console.log("");
						console.log("======= NEW (accurate) =======");
						console.log(coloredText(line.trim()));
						console.log("==============================");
						console.log("");
						console.log("======= OLD (guessed, similarity: " + totalLine.similarity + ") =======");
						console.log(coloredText(oldFileTextLines[totalLine.line].trim()));
						console.log("==============================");

						let similarAnswer = await SimilarityConfirm.run();
						if (similarAnswer) {
							nativeResult = {
								oldLine: totalLine.line,
								newLine: newLinesCount,
								oldLineTxt: oldFileTextLines[totalLine.line],
								newLineTxt: line
							};
							break;
						}
					}
				}

			}
			newLinesCount++;
		}

		return nativeResult;
	} else {
		fs.appendFileSync("errorhashes.txt", native + "\n");
		console.error("Warning! Native " + native + " not found in any script!");
	}
}

// The same as above, but without function/array stuff of course
const replaceChangingStuffRegex = /(?:\/\*(\d+)\*\/)|(?:\.f_(\d+))|(?:Global_(\d+))|(?:Local_(\d+))/g;
// Group 1 = Native Function Name (incl. potential namespace)
const nativeFunctionRegex = /((?:(?:unk_)|(?:\w+::))[0-9a-zA-Z_]+)\(/g;

function findHashByNamespaceNative(namespace, native) {
	for (let nativeNamespace of nativeNamespaces) {
		if (nativeNamespace.namespace == namespace) {
			for (let nativeName of nativeNamespace.natives) {
				if (nativeName.name == native)
					return nativeName.hash;
			}
		}
	}
	console.error("Warning! No hash found for native " + namespace + "::" + native);
}

(async () => {
	for (let native of natives) {
		let nativeRes = await checkNative(native);
		if (nativeRes !== undefined) {
			let oldNativeLine = nativeRes.oldLineTxt.replace(replaceChangingStuffRegex, "");
			let newNativeLine = nativeRes.newLineTxt.replace(replaceChangingStuffRegex, "");

			let nativeMatchCount = 0;
			for (let newLineMatches of matchAll(newNativeLine, nativeFunctionRegex)) {
				if (newLineMatches[1].includes("unk_" + native)) {
					break;
				}
				nativeMatchCount++;
			}

			let nativeMatchCountO = 0;
			for (let oldLineMatches of matchAll(oldNativeLine, nativeFunctionRegex)) {
				if (nativeMatchCountO == nativeMatchCount) {
					let nativeFunctionName = oldLineMatches[1];
					console.log(native + " translates to function " + oldLineMatches[1]);

					if (nativeFunctionName.startsWith("unk_")) {
						console.log(greenText(native + " => " + nativeFunctionName.replace("unk_", "")));
						fs.appendFileSync("crosstable.txt", "{" + nativeFunctionName.replace("unk_", "") + ", " + native + "},\n");
					} else {
						let nativeNamespace = nativeFunctionName.split("::")[0];
						let nativeName = nativeFunctionName.split("::")[1];
						let nativeHash = findHashByNamespaceNative(nativeNamespace, nativeName);
						console.log(greenText(native + " => " + nativeHash));
						fs.appendFileSync("crosstable.txt", "{" + nativeHash + ", " + native + "},\n");
					}
					break;
				}
				nativeMatchCountO++;
			}
		} else {
			console.error("Warning! Native " + native + " has no result!!!");
			fs.appendFileSync("errorhashes.txt", native + "\n");
		}
	}
})();