The binary was packed by UPX like the original packer for obfuscation. First, we need to unpack the binary, so we will use gdb to dump memory for analyzing the binary.
% gdb optimized
gdb-peda$ r
Starting program: /mnt/hgfs/ctf/tsgctf/rev/optimized/optimized
Enter password: ^C
Program received signal SIGINT, Interrupt.
<snip>
gdb-peda$ dump memory unpacked 0x00400000 0x00604000
Here is the decompled code after analysis:
int __cdecl main(int argc, const char **argv, const char **envp)
{
int v3; // edi
__int64 v4; // rbx
__int64 v5; // rax
const char *v6; // rdi
int v8; // [rsp+Ch] [rbp-12Ch] BYREF
unsigned int pass1; // [rsp+10h] [rbp-128h] BYREF
unsigned int pass2; // [rsp+14h] [rbp-124h] BYREF
unsigned int pass3; // [rsp+18h] [rbp-120h] BYREF
unsigned int pass4; // [rsp+1Ch] [rbp-11Ch] BYREF
unsigned int v13; // [rsp+20h] [rbp-118h]
unsigned int v14; // [rsp+24h] [rbp-114h]
unsigned int v15; // [rsp+28h] [rbp-110h]
unsigned int v16; // [rsp+2Ch] [rbp-10Ch]
char flag[264]; // [rsp+30h] [rbp-108h] BYREF
printf("Enter password: ");
sub_400820(qword_6040A8);
if ( scanf("%u %u %u %u", &pass1, &pass2, &pass3, &pass4) == 4 )
{
if ( _mm_movemask_epi8(
_mm_cmpeq_epi8(
_mm_slli_si128(0x9569uLL, 8),
_mm_slli_si128((((0x2AF91 * (0x5F50DDCA7B17LL * pass1)) >> 64) & 0x3FFFF), 8))) == 0xFFFF
&& _mm_movemask_epi8(
_mm_cmpeq_epi8(
_mm_slli_si128(0x26CF2uLL, 8),
_mm_slli_si128((((0x34AB9 * (0x4DC4591DAC8FLL * pass1)) >> 64) & 0x3FFFF), 8))) == 0xFFFF
&& _mm_movemask_epi8(
_mm_cmpeq_epi8(
_mm_slli_si128(0x20468uLL, 8),
_mm_slli_si128((((0x36B39 * (0x4AE11552DF1ALL * pass2)) >> 64) & 0x3FFFF), 8))) == 0xFFFF
&& _mm_movemask_epi8(
_mm_cmpeq_epi8(
_mm_slli_si128(0x3787AuLL, 8),
_mm_slli_si128((((0x3A2D3 * (0x46680B140EFFLL * pass2)) >> 64) & 0x3FFFF), 8))) == 0xFFFF
&& (v3 = pass3, 0x4D935BBD3E0LL * pass3 < 0x4D935BBD3E0LL)
&& _mm_movemask_epi8(
_mm_cmpeq_epi8(
_mm_slli_si128(0x5563uLL, 8),
_mm_slli_si128((((0x27DF9 * (0x66B9B431B9EDLL * pass3)) >> 64) & 0x3FFFF), 8))) == 0xFFFF
&& 0x1E5D2BE81C5LL * pass4 < 0x1E5D2BE81C5LL
&& _mm_movemask_epi8(
_mm_cmpeq_epi8(
_mm_slli_si128(0x133E7uLL, 8),
_mm_slli_si128((((0x3BC65 * (0x448626500938LL * pass4)) >> 64) & 0x3FFFF), 8))) == 0xFFFF )
{
v13 = pass1;
v14 = pass2;
v15 = pass3;
v16 = pass4;
v4 = EVP_CIPHER_CTX_new();
if ( !v4
|| (v5 = EVP_chacha20(), v3 = v4, EVP_DecryptInit_ex(v4, v5, 0LL, &pass1, "welcometotsgctfaaa") != 1)
|| (v3 = v4, EVP_DecryptUpdate(v4, flag, &v8, qword_403420, 20LL) != 1)
|| (v3 = v4, EVP_DecryptFinal_ex(v4, &flag[v8], &v8) != 1) )
{
exit(v3);
}
EVP_CIPHER_CTX_free(v4);
v6 = flag;
}
else
{
v6 = "Wrong!";
}
}
else
{
v6 = "Bad format!";
}
puts(v6);
return 0;
}
Solver:
from z3 import *
s = Solver()
pass1 = BitVec('pass1', 32)
pass2 = BitVec('pass2', 32)
pass3 = BitVec('pass3', 32)
pass4 = BitVec('pass4', 32)
# Note: these takes a lot of times so you can execute individually.
#s.add(0x9569 == LShR((0x2AF91 * ZeroExt(64, 0x5F50DDCA7B17 * ZeroExt(32, pass1))), 64) & 0x3FFFF)
#s.add(0x26CF2 == LShR((0x34AB9 * ZeroExt(64, 0x4DC4591DAC8F * ZeroExt(32, pass1))), 64) & 0x3FFFF)
#s.add(0x20468 == LShR((0x36B39 * ZeroExt(64, 0x4AE11552DF1A * ZeroExt(32, pass2))), 64) & 0x3FFFF)
#s.add(0x3787A == LShR((0x3A2D3 * ZeroExt(64, 0x46680B140EFF * ZeroExt(32, pass2))), 64) & 0x3FFFF)
#s.add((0x4D935BBD3E0 * ZeroExt(64, pass3)) & 0xffffffffffffffff < 0x4D935BBD3E0)
#s.add(0x5563 == LShR(0x27DF9 * ZeroExt(64, (0x66B9B431B9ED * ZeroExt(32, pass3))), 64) & 0x3FFFF)
s.add((0x1E5D2BE81C5 * ZeroExt(64, pass4)) & 0xffffffffffffffff < 0x1E5D2BE81C5)
s.add(0x133E7 == LShR((0x3BC65 * ZeroExt(64, 0x448626500938 * ZeroExt(32, pass4))), 64) & 0x3FFFF)
assert s.check() == sat
m = s.model()
print(m)
print(m[pass1], m[pass2], m[pass3], m[pass4])
% ./optimized
Enter password: 772928896 2204180909 4273479145 1334930147
TSGCTF{F457_m0dul0!}@