cw2k/Bethesda_unpack.py

## Bethesda_unpack.py
"""

Bethesda.net extrator script
============================

So what is that?
Elder scrolls online comes with that installer:
Bethesda.net_Launcher.exe
it is about 25 MB.
24MB of the file is a zip.
But extracting is not possible because it is password protected.
The Launcher.exe somehow 'knows' the password.

I reverse engieered how that works.
Each file uses the ExtraData field in the Zip header to store the password

Offset      0  1  2  3  4  5  6  7   8  9 10 11 12 13 14 15

00000000   50 4B 03 04 14 00 0B 00  08 00 18 49 5B 55 00 00   PK         I[U
00000016   00 00 00 00 00 00 00 00  00 00 1F 00 10 00 66 6F                 fo
00000032   6E 74 73 2F 6C 65 61 67  75 65 5F 67 6F 74 68 69   nts/league_gothi
00000048   63 2D 77 65 62 66 6F 6E  74 2E 65 6F 74 10 88 0C   c-webfont.eot ˆ
00000064   00 C4 EE 37 F7 51 4B 8E  12 AE C2 72 8A            Äî7÷QKŽ ®ÂrŠ

Zip-ExtraData:
10 88 0C 00 C4 EE 37 F7 51 4B 8E 12 AE C2 72 8A
0x8810 Magic
       000C Length
            45 70 3B 3F 61 6B 4E 33 2E 42 72 4B    Ep;?akN3.BrK     Password transformed

Next time I look for the 'SetZipPassword' it is helpfull to know that it's near these consts:
  (*Keys)[0] = 0x12345678; # 305419896
  (*Keys)[1] = 0x23456789; # 591751049
  (*Keys)[2] = 0x34567890; # 878082192


  pass = *Password;

  "Password-based encryption in ZIP files":
  https://courses.cs.ut.ee/MTAT.07.022/2015_fall/uploads/Main/dmitri-report-f15-16.pdf

So in IDA Pro or Ollydbg just search for const 0x23456789 to find the SetZipPassword functions.

"UnzipperStruct:..\\..\\..\\..\\shared\\unzipper.c"


How to use that script?
0. Pust script file into dir with Bethesda.net_Launcher.exe
1. Split of the Exe stub from the Bethesda.net_Launcher.exe and save it as Bethesda.net_Launcher.zip
2. run in cmd.exe run "python.exe Bethesda_unpack.py"

Extraction is pretty slow. If interupt take care for cleaning 0-byte files manually.
For debugging comment in some of the '#print '

You may rezip the file and merge it with the exe-stub.
Happy modding.
"""
import zipfile
class myBinaryReader(bytearray):

    offset = 0

    storedPos = 0
    def PosStore(_):    _.storedPos = _.offset
    def PosRestore(_):  _.offset    = _.storedPos

    def isEOS(_): return (_.offset >= len(_))
    def BytesLeft(_): return ( len(_) - _.offset )

    def matches(_, data):
        return _.read( len(data) ) == data


    __float64_Size = 8
    def float64(_): return unpack_from(
        "<d",
        _.read( _.__float64_Size)) [0]

    __Int8_Size  = 1
    def u8(_): return _.uInt(_.__Int8_Size )
    def i8(_): return _ .Int(_.__Int8_Size )

    __Int16_Size = 2
    def u16(_): return _.uInt(_.__Int16_Size)
    def i16(_): return  _.Int(_.__Int16_Size)

    __Int32_Size = 4
    def u32(_): return _.uInt(_.__Int32_Size)
    def i32(_): return  _.Int(_.__Int32_Size)

    def u32_bigEndian(_): return _._Int_bigEndian( _.__Int32_Size, False )

    __Int64_Size = 8
    def u64(_): return _.uInt(_.__Int64_Size)
    def i64(_): return  _.Int(_.__Int64_Size)


    def Int(_, len ):
        return _._Int( len, True)

    def uInt(_, len ):
        return _._Int( len, False)

    def _Int(_, len, signed):
        return int.from_bytes( _.read( len) ,
                               'little', signed=signed)

    def _Int_bigEndian(_, len, signed):
        return int.from_bytes( _.read( len) ,
                               'big', signed=signed)

    def read(_, len) :
        data = _[_.offset: _.offset + len ]
        _.offset += len
        return data


def makePrintable( Datareader ):
    passwd = b''

    DataSize = Datareader.u16()

    for i in  range(DataSize):
        val    = Datareader.u8()
        if ( val == 0 ):
            #print ( f'Exiting loop due to nullByte countered @{i}/{DataSize}' )
            break


        add    = (   1 << (i % 32)  ) & 0xff  #  1, 2, 4, 8, 10, 20, 40, 80, 0, 0 ...
                                              # note: 'mod 32' is there to simulate overflow effect
                                              # MOV     BL, 1
                                              # SHL     BL, CL


        newVal = ( val + add  ) & 0x7f

        # Limit newVal to range 0x21..0x7E
        newVal &= 0x7f  if ( newVal != 0x7f ) else 0x3F

        if ( newVal < 0x21 ):
            newVal |= (1 << (newVal % 3 + 5) )  # 0x20 0x40 0x80  0x20 0x40 0x80  ...
            newVal += 1

        passwd_char = chr (newVal)

        passwd += newVal.to_bytes(1, "little")

        #print ( f'{add :0>2X}  + {val :0>2X} => {newVal :0>2X} \t {passwd_char} ' )

    #passwd = passwd.decode('iso8859_1')


    return passwd

#Testing
ZipExtraData = 0x10880C00C4EE37F7514B8E12AEC2728A #Ep;?akN3.BrK
ZipExtraData = 0x10880C00452E8B6F2A2F36655AC7058E #F00w:OveZG† b'F00w:OveZG\x86\x8f'
# finding the right 'decode'
# 0x86 -> 134 ->ALT+num 1-3-4 ->å
# https://tripleee.github.io/8bit/#86
# å	(U+00E5)	cp437, cp775, cp850, cp857, cp858, cp861, cp865
# So we may try: passwd.decode('cp850')


CentralFile_hdr_sig = b'PK\x01\x02' # 0x02014b50
ExtraData_Hdr = 0x8810


zipfileName = "c://temp//Bethesda.net_Launcher//ProgramData//Host.c0da3c1c5bf82ed6ac9afcd89b1f79b8c99c88bb//061b8d0af8fa3d892fdc9723a00d5d6ccff18f5a.patchmanifest2"
zipfileName = "Bethesda.net_Launcher.zip"
extractpath = "./Bethesda.net_Launcher"

#Datareader = myBinaryReader(ZipExtraData.to_bytes(0x10,"big"))
#HeaderID = Datareader.u16()
#print (passwd)
#passwd = passwd.decode('latin-1')
#print( f'  { passwd :<100}', f'{FileName.decode()}' )
#quit

# Get CentralFile_header
f = open(zipfileName, "rb")
Files =  f.read().split( CentralFile_hdr_sig )[1:]
f.close()

ExtraSizeMax = 0
for file in Files:
    Datareader = myBinaryReader( file) #.split(ExtraData_Hdr.to_bytes(2,"little"))[1] )
    version = Datareader.u32()
    gpFlags = Datareader.u32()
    time = Datareader.u32()
    CRC32 = Datareader.u32()
    Size_comp = Datareader.u32()
    Size_decomp = Datareader.u32()


    FileNameSize = Datareader.u16()
    ExtraSize = Datareader.u16()
    CmtSize = Datareader.u16()

    dummy = Datareader.u32()
    dummy = Datareader.u32()
    dummy = Datareader.u32()

    FileName =  Datareader.read(FileNameSize)
    FileName =FileName.decode()


   # Datareader = myBinaryReader(ZipExtraData.to_bytes(0x10,"big"))

    HeaderID = Datareader.u16()
    isExpectedExtraData = (ExtraData_Hdr == HeaderID)

    assert (isExpectedExtraData)

    #ExtraSizeMax = max( ExtraSizeMax, ExtraSize )
    #print ("ExtraSize: %i \t curExtraSizeMax: %i" % (ExtraSize, ExtraSizeMax))


    passwd = makePrintable( Datareader )

    #print (passwd)
    import os.path
    if not os.path.exists( extractpath + "/" + FileName):

        try:
            with zipfile.ZipFile(zipfileName, mode="r") as archive:
                #archive.printdir()
                archive.extract(FileName, extractpath , passwd )

        except Exception as error:
            print(error, passwd, passwd.decode('cp850') )

    passwd = passwd.decode('cp850')
    print( f'  { passwd :<100}', f'{FileName}' )
	"""

	Bethesda.net extrator script
	============================

	So what is that?
	Elder scrolls online comes with that installer:
	Bethesda.net_Launcher.exe
	it is about 25 MB.
	24MB of the file is a zip.
	But extracting is not possible because it is password protected.
	The Launcher.exe somehow 'knows' the password.

	I reverse engieered how that works.
	Each file uses the ExtraData field in the Zip header to store the password

	Offset 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15

	00000000 50 4B 03 04 14 00 0B 00 08 00 18 49 5B 55 00 00 PK I[U
	00000016 00 00 00 00 00 00 00 00 00 00 1F 00 10 00 66 6F fo
	00000032 6E 74 73 2F 6C 65 61 67 75 65 5F 67 6F 74 68 69 nts/league_gothi
	00000048 63 2D 77 65 62 66 6F 6E 74 2E 65 6F 74 10 88 0C c-webfont.eot ˆ
	00000064 00 C4 EE 37 F7 51 4B 8E 12 AE C2 72 8A Äî7÷QKŽ ®ÂrŠ

	Zip-ExtraData:
	10 88 0C 00 C4 EE 37 F7 51 4B 8E 12 AE C2 72 8A
	0x8810 Magic
	000C Length
	45 70 3B 3F 61 6B 4E 33 2E 42 72 4B Ep;?akN3.BrK Password transformed

	Next time I look for the 'SetZipPassword' it is helpfull to know that it's near these consts:
	(*Keys)[0] = 0x12345678; # 305419896
	(*Keys)[1] = 0x23456789; # 591751049
	(*Keys)[2] = 0x34567890; # 878082192


	pass = *Password;

	"Password-based encryption in ZIP files":
	https://courses.cs.ut.ee/MTAT.07.022/2015_fall/uploads/Main/dmitri-report-f15-16.pdf

	So in IDA Pro or Ollydbg just search for const 0x23456789 to find the SetZipPassword functions.

	"UnzipperStruct:..\\..\\..\\..\\shared\\unzipper.c"



	How to use that script?
	0. Pust script file into dir with Bethesda.net_Launcher.exe
	1. Split of the Exe stub from the Bethesda.net_Launcher.exe and save it as Bethesda.net_Launcher.zip
	2. run in cmd.exe run "python.exe Bethesda_unpack.py"

	Extraction is pretty slow. If interupt take care for cleaning 0-byte files manually.
	For debugging comment in some of the '#print '

	You may rezip the file and merge it with the exe-stub.
	Happy modding.
	"""
	import zipfile
	class myBinaryReader(bytearray):

	offset = 0

	storedPos = 0
	def PosStore(_): _.storedPos = _.offset
	def PosRestore(_): _.offset = _.storedPos

	def isEOS(_): return (_.offset >= len(_))
	def BytesLeft(_): return ( len(_) - _.offset )

	def matches(_, data):
	return _.read( len(data) ) == data



	__float64_Size = 8
	def float64(_): return unpack_from(
	"<d",
	_.read( _.__float64_Size)) [0]

	__Int8_Size = 1
	def u8(_): return _.uInt(_.__Int8_Size )
	def i8(_): return _ .Int(_.__Int8_Size )

	__Int16_Size = 2
	def u16(_): return _.uInt(_.__Int16_Size)
	def i16(_): return _.Int(_.__Int16_Size)

	__Int32_Size = 4
	def u32(_): return _.uInt(_.__Int32_Size)
	def i32(_): return _.Int(_.__Int32_Size)

	def u32_bigEndian(_): return _._Int_bigEndian( _.__Int32_Size, False )

	__Int64_Size = 8
	def u64(_): return _.uInt(_.__Int64_Size)
	def i64(_): return _.Int(_.__Int64_Size)


	def Int(_, len ):
	return _._Int( len, True)

	def uInt(_, len ):
	return _._Int( len, False)

	def _Int(_, len, signed):
	return int.from_bytes( _.read( len) ,
	'little', signed=signed)

	def _Int_bigEndian(_, len, signed):
	return int.from_bytes( _.read( len) ,
	'big', signed=signed)

	def read(_, len) :
	data = _[_.offset: _.offset + len ]
	_.offset += len
	return data



	def makePrintable( Datareader ):
	passwd = b''

	DataSize = Datareader.u16()

	for i in range(DataSize):
	val = Datareader.u8()
	if ( val == 0 ):
	#print ( f'Exiting loop due to nullByte countered @{i}/{DataSize}' )
	break


	add = ( 1 << (i % 32) ) & 0xff # 1, 2, 4, 8, 10, 20, 40, 80, 0, 0 ...
	# note: 'mod 32' is there to simulate overflow effect
	# MOV BL, 1
	# SHL BL, CL



	newVal = ( val + add ) & 0x7f

	# Limit newVal to range 0x21..0x7E
	newVal &= 0x7f if ( newVal != 0x7f ) else 0x3F

	if ( newVal < 0x21 ):
	newVal \|= (1 << (newVal % 3 + 5) ) # 0x20 0x40 0x80 0x20 0x40 0x80 ...
	newVal += 1

	passwd_char = chr (newVal)

	passwd += newVal.to_bytes(1, "little")

	#print ( f'{add :0>2X} + {val :0>2X} => {newVal :0>2X} \t {passwd_char} ' )

	#passwd = passwd.decode('iso8859_1')


	return passwd

	#Testing
	ZipExtraData = 0x10880C00C4EE37F7514B8E12AEC2728A #Ep;?akN3.BrK
	ZipExtraData = 0x10880C00452E8B6F2A2F36655AC7058E #F00w:OveZG† b'F00w:OveZG\x86\x8f'
	# finding the right 'decode'
	# 0x86 -> 134 ->ALT+num 1-3-4 ->å
	# https://tripleee.github.io/8bit/#86
	# å (U+00E5) cp437, cp775, cp850, cp857, cp858, cp861, cp865
	# So we may try: passwd.decode('cp850')


	CentralFile_hdr_sig = b'PK\x01\x02' # 0x02014b50
	ExtraData_Hdr = 0x8810


	zipfileName = "c://temp//Bethesda.net_Launcher//ProgramData//Host.c0da3c1c5bf82ed6ac9afcd89b1f79b8c99c88bb//061b8d0af8fa3d892fdc9723a00d5d6ccff18f5a.patchmanifest2"
	zipfileName = "Bethesda.net_Launcher.zip"
	extractpath = "./Bethesda.net_Launcher"

	#Datareader = myBinaryReader(ZipExtraData.to_bytes(0x10,"big"))
	#HeaderID = Datareader.u16()
	#print (passwd)
	#passwd = passwd.decode('latin-1')
	#print( f' { passwd :<100}', f'{FileName.decode()}' )
	#quit

	# Get CentralFile_header
	f = open(zipfileName, "rb")
	Files = f.read().split( CentralFile_hdr_sig )[1:]
	f.close()

	ExtraSizeMax = 0
	for file in Files:
	Datareader = myBinaryReader( file) #.split(ExtraData_Hdr.to_bytes(2,"little"))[1] )
	version = Datareader.u32()
	gpFlags = Datareader.u32()
	time = Datareader.u32()
	CRC32 = Datareader.u32()
	Size_comp = Datareader.u32()
	Size_decomp = Datareader.u32()


	FileNameSize = Datareader.u16()
	ExtraSize = Datareader.u16()
	CmtSize = Datareader.u16()

	dummy = Datareader.u32()
	dummy = Datareader.u32()
	dummy = Datareader.u32()

	FileName = Datareader.read(FileNameSize)
	FileName =FileName.decode()


	# Datareader = myBinaryReader(ZipExtraData.to_bytes(0x10,"big"))

	HeaderID = Datareader.u16()
	isExpectedExtraData = (ExtraData_Hdr == HeaderID)

	assert (isExpectedExtraData)

	#ExtraSizeMax = max( ExtraSizeMax, ExtraSize )
	#print ("ExtraSize: %i \t curExtraSizeMax: %i" % (ExtraSize, ExtraSizeMax))


	passwd = makePrintable( Datareader )

	#print (passwd)
	import os.path
	if not os.path.exists( extractpath + "/" + FileName):

	try:
	with zipfile.ZipFile(zipfileName, mode="r") as archive:
	#archive.printdir()
	archive.extract(FileName, extractpath , passwd )

	except Exception as error:
	print(error, passwd, passwd.decode('cp850') )

	passwd = passwd.decode('cp850')
	print( f' { passwd :<100}', f'{FileName}' )