extract password-protected InstallBuilder installer

extract-installbuilder.tcl

#!./tclkit

## prepare runtime environment
proc init {} {
	## mount optional.pak (for tcltwofish)
	set optionalPak installbuilder/paks/optional.pak
	vfs::mk4::Mount $optionalPak $optionalPak -readonly

	## adjust library search path
	set ::auto_path [list $tcl::kitpath/lib/tcl$::tcl_version $tcl::kitpath/lib $tcl::kitpath/libraries $optionalPak/linux-x64 $tcl::kitpath]

	## load packages
	package require maui::util
	package require vfs::cookfs
}

proc extract_metakit {src dst} {
	## mount metakit
	set db [tclKitMkOpen $src]
	vfs::filesystem mount $src [list vfs::mk4::handler $db]
	vfs::RegisterMount $src [list vfs::mk4::Unmount $db]

	## ensure destination directory
	file mkdir $dst

	## read cookfsinfo.txt
	set f [open $src/cookfsinfo.txt]
	gets $f cookfsinfo
	close $f

	## read manifest.txt
	set f [open $src/manifest.txt]
	set manifest [read $f]
	close $f

	## copy project.xml
	file copy -- $src/project.xml $dst

	## copy images/
	file copy -- $src/images $dst

	## unmount metakit
	vfs::filesystem unmount $src

	return [list $cookfsinfo $manifest]
}

proc extract {src password dst} {
	## clean up destination
	file delete -force $dst

	## copy images/ to dst, get manifest, mount cookfs
	lassign [extract_metakit $src $dst] cookfsinfo manifest

	## mount cookfs
	set cookfsHandle [vfs::cookfs::Mount $src $src -readonly {*}$cookfsinfo]

	## prepare decryption
	set payloadinfo [$cookfsHandle getmetadata "installbuilder.payloadinfo"]
	maui::util::m4ZQu $password $payloadinfo

	## copy cookfs content to components/
	set dst $dst/components

	## extract directory/file/link
	foreach {fileName props} $manifest {
		puts "$fileName"

		# directory  mode    mtime  0   {0  0}
		# file       mode    mtime  0  {{0  size} {0 size}}
		# link       target  mtime      {0  0}
		lassign $props type arg mtime
		switch -- $type {
			directory {
				file mkdir $dst/$fileName
				file attributes $dst/$fileName -permissions $arg
			}
			file {
				file mkdir [file dirname $dst/$fileName]
				file copy $src/$fileName $dst/$fileName
				set numParts [llength [lindex $props 4]]
				if {$numParts > 0} {
					set fdst [open $dst/$fileName {WRONLY APPEND BINARY}]
					for {set i 1} {$i < $numParts} {incr i} {
						set fsrc [open $src/${fileName}___bitrockBigFile$i {RDONLY BINARY}]
						fcopy $fsrc $fdst
						close $fsrc
					}
					close $fdst
				}
				file attributes $dst/$fileName -permissions $arg
				file mtime $dst/$fileName $mtime
			}
			link {
				file mkdir [file dirname $dst/$fileName]
				# file link -symbolic $dst/$fileName $arg
				# file mtime $dst/$fileName $mtime
				exec ln -s -- $arg $dst/$fileName
				exec touch -h -t [clock format $mtime -format %Y%m%d%H%M.%S] $dst/$fileName
			}
			default {
				error "not implemented"
			}
		}
	}

	## restore directory mtime
	foreach {fileName props} $manifest {
		lassign $props type arg mtime
		if {$type == "directory"} {
			file mtime $dst/$fileName $mtime
		}
	}

	## unmount cookfs
	vfs::filesystem unmount $src
}

init
extract {*}$argv

get-tclkit.sh

#!/bin/sh
cp installbuilder/paks/linux-x64-noupx.pak tclkit
sed --in-place --null-data --expression='/^if {\[file isfile \[file join \$::tcl::kitpath main.tcl/{s/^./\x1A/}' tclkit
chmod +x tclkit

[21:05:43]: ::maui::util::m4ZQu result:
00000000: 84 DB 68 7D 1F BF 46 7A  26 85 1B EE AB B7 AB 7D | ..h}..Fz&......}
00000010: B0 1E 9C 30 2A 2A 2D FB  C8 73 0F 20 29 6F 42 74 | ...0**-..s. )oBt
[21:08:02]: ::maui::util::m4ZQu result:
00000000: 49 6E 76 61 6C 69 64 20  70 61 79 6C 6F 61 64 20 | Invalid payload 
00000010: 70 61 73 73 77 6F 72 64                          | password

thanks, this command is the exact function I am looking for. it's contains in obfuscated bytecode of maui-util.tcl.
hope it only have sha2 and twofish dependency.
I'll try to finger-out the algorithm for m4ZQu, then port them to cuda version

::maui::util::a64bL {password key iv times} { #hash times
	set YKL8w [sha2::sha256 -bin $password]
	for {set i 0} {$i < $times} {incr i} {
		set YKL8w [tcltwofish::encrypt $key $YKL8w iv]
	}
	return [sha2::sha256 -bin $YKL8w]
}

proc maui::util::m4ZQu {password LZQjH} { #check password
	set ::maui::util::H6VeX ""
	set ::maui::util::EL0BW [string repeat \0 32]

	package require tcltwofish
	package require sha256

	if {[string length $LZQjH] == 0} {
		error "Unable to initialize encryption - missing essential data"
	}
	

	if {[binary scan $LZQjH Ia16a32a64a32a* times iv passwordKey encryptedKey payloadIVsHash encryptedPayloadIVs] < 6} {
		error "Unable to initialize encryption - header missing or corrupt"
	}


	set iHcWR [a64bL $password $passwordKey $iv $times]
	set BJvoG [string range [tcltwofish::decrypt $iHcWR $encryptedKey] 32 63]
	

	for {set i 0} {$i < $times} {incr i 64} {
		set encryptedPayloadIVs [tcltwofish::decrypt $BJvoG $encryptedPayloadIVs]
	}
	set ZQpQw [string range $encryptedPayloadIVs 32 end]


	if {[sha2::sha256 -bin $ZQpQw] != $payloadIVsHash} {
		maui::d_RUj "Invalid payload password"
	}

	set ::maui::util::H6VeX $ZQpQw
	set ::maui::util::EL0BW $BJvoG
}

proc ::maui::util::MI_oJ {compressionAlgorithm data} { #decrypt and decompress
	if {$compressionAlgorithm == "0"} {
		set compressionAlgorithm zip
	}  elseif {$compressionAlgorithm == "1"} {
		set compressionAlgorithm lzma
	}
	if {[catch {
		if {[binary scan $data Ica* checksum iv data] < 3} {
			error "Unable to decrypt payload header"
		}
		set iv [expr {$iv & 0xff}]
		set iv [string range $::maui::util::H6VeX [expr {$iv * 16}] [expr {$iv * 16 + 15}]]
		set data [tcltwofish::decrypt $::maui::util::EL0BW $data iv]

		set hbA8n [format 0x%08x [expr {[zlib crc32 $data] & 0xffffffff}]]
		set checksum [format 0x%08x [expr {$checksum & 0xffffffff}]]
		if {$hbA8n != $checksum} {
			error "Unable to decrypt payload - CRC mismatch"
		}
		set data [QZqgk [string range $data 32 end]]
		if {($compressionAlgorithm == "lzma") || ($compressionAlgorithm == "lzma-ultra")} {
			set YKL8w [lzmadec $data]
		}  elseif {($compressionAlgorithm == "lzham") || ($compressionAlgorithm == "lzham-ultra")} {
			set YKL8w [tcllzham::decompress $data]
		}  else  {
			set YKL8w [vfs::zip -mode decompress -nowrap 1 $data]
		}
	} V77Ls]} {
		set ykaki $::errorCode
		set qXoNi $::errorInfo
		maui::util::debug "decryptPayload: error decrypting: $V77Ls"
		error $V77Ls $qXoNi $ykaki
	}
	return $YKL8w
}

thank you @jon1scr, today I placed a logger on TclEvalObjvInternal and found (sha256->encrypt*16000->sha256) is belong of a64bL, too.

209 binary +++++++
209 binary -------
1 a64bL +++++++
1 sha2::sha256 +++++++
::sha2::sha256 buflen: 11, buf:
00000000: 7A 31 32 33 34 35 36 37  38 39 30                | z1234567890
1 ::sha2::_sha256 +++++++
8346 array +++++++
8346 array -------
........
1183 unset +++++++
1183 unset -------
2 SHA256Final -------
2 ::sha2::_sha256 -------
sha256 hash:
00000000: 50 8A 8C 5F EE 98 49 AA  1A 9B 85 9A BA 19 4D 67 | P.._..I.......Mg
00000010: FA 4D A2 D3 58 4E 7F 4C  32 59 A4 B7 59 FC 96 04 | .M..XN.L2Y..Y...
2 sha2::sha256 -------
1 a64bL -------
1 tcltwofish::decrypt +++++++
::tcltwofish::decrypt buflen: 64, buf:
00000000: 05 8A 77 A1 15 85 96 D3  62 33 04 CC F3 2D B4 67 | ..w.....b3...-.g
00000010: C1 0E 28 6A 2C A9 EF 30  B4 BC 67 86 B2 F9 D1 2D | ..(j,..0..g....-
00000020: F6 29 C3 9F 4E EB 33 AE  7C 28 96 B6 F0 C2 4F 58 | .)..N.3.|(....OX
00000030: 02 B9 4A D7 21 30 E2 37  62 C3 60 45 65 42 46 AF | ..J.!0.7b.`EeBF.
1 tcltwofish::decrypt -------

I'll try verify your decompiled code on known password installers.
PS: how do you decompile the bytecode from maui-utils, is there any tools sugguested?

Tbc is not good at protecting string literals. They can be decoded directly.

I think use installbuilder to build a empty installer with payload encrypt option enabled, then the built installer's metakit will have tcltwofish now.

@banxian You can always extract pre-built tcltwofish for win/linux/mac from optional.pak under InstallBuilder installation directory.

Infact I compared metakit content between built installer and windows-noupx.pak, the installer contains all package plus tcltwofish and 8 extra project files than windows-noupx.pak.
I mean it's a better start point which does not require additional optional.pak dependency, so it's easy to use tcltksh to pack the script and runtime to a single executable.