Task 10 - equation

snippet.cpp

// offset: 0x9c3
int __usercall sub_9C3@<eax>(bn *a1@<eax>, bn *a2@<edx>, bn *a3@<ecx>, bn *_arg_1)
{
  bn *arg_1; // edi
  bn *arg_2; // esi
  int result; // eax
  struct bn val_res; // [esp+0h] [ebp-284h]
  struct bn val_2; // [esp+80h] [ebp-204h]
  struct bn val_1; // [esp+100h] [ebp-184h]
  struct bn val_0; // [esp+180h] [ebp-104h]
  struct bn div_result; // [esp+200h] [ebp-84h]
  bn *div_by; // [esp+280h] [ebp-4h]

  arg_1 = a1;
  arg_2 = a2;
  div_by = a3;
  
  bignum_init(&val_res);
  bignum_from_int(_arg_1, 1, 0);
  bignum_from_int(&val_2, 2, 0);
  bignum_from_int(&val_1, 1, 0);
  bignum_from_int(&val_0, 0, 0);
  
  bignum_divmod(arg_2, &val_2, &div_result, &val_res);
  
  if ( val_res.array[31] == val_1.array[31] )
  {
    bignum_assign(_arg_1, arg_1);
    bignum_div((int)arg_2, (int)&val_2, val_res.array);
    bignum_assign(arg_2, &val_res);
    
    result = val_0.array[31];
    if ( arg_2->array[31] == val_0.array[31] )
      return result;
    
    bignum_mul((char *)arg_1, (int)arg_1, &val_res);
    bignum_divmod(&val_res, div_by, &div_result, arg_1);
  }
  
  while ( 1 )
  {
    bignum_divmod(arg_2, &val_2, &div_result, &val_res);
    if ( val_res.array[31] != val_0.array[31] )
    {
      bignum_mul((char *)_arg_1, (int)arg_1, &val_res);
      bignum_divmod(&val_res, div_by, &div_result, _arg_1);
    }
    bignum_div((int)arg_2, (int)&val_2, val_res.array);
    bignum_assign(arg_2, &val_res);
    
    result = val_0.array[31];
    if ( arg_2->array[31] == val_0.array[31] )
      break;
    
    bignum_mul((char *)arg_1, (int)arg_1, &val_res);
    bignum_divmod(&val_res, div_by, &div_result, arg_1);
  }
  
  return result;
}

// offset: 0xdbe
void __cdecl __noreturn main_func(int a1, char *buffer, int buf_len)
{
  int fd; // esi
  char data; // [esp+0h] [ebp-A4Ch]
  int flag_res; // [esp+18h] [ebp-A34h]
  bn key_part1; // [esp+44h] [ebp-A08h]
  bn key_part2; // [esp+C4h] [ebp-988h]
  bn const_0; // [esp+144h] [ebp-908h]
  bn const_1; // [esp+1C4h] [ebp-888h]
  bn div_reminder; // [esp+244h] [ebp-808h]
  bn const_3; // [esp+2C4h] [ebp-788h]
  bn a2; // [esp+344h] [ebp-708h]
  bn internal_buf; // [esp+3C4h] [ebp-688h]
  bn out_buf0; // [esp+444h] [ebp-608h]
  bn out_buf1; // [esp+4C4h] [ebp-588h]
  bn r_buf; // [esp+544h] [ebp-508h]
  bn div_res; // [esp+5C4h] [ebp-488h]
  signed int a4; // [esp+644h] [ebp-408h]
  int base; // [esp+A44h] [ebp-8h]
  pid_t pid; // [esp+A48h] [ebp-4h]

  base = 0;
  pid = MEMORY[0x81A5280];
  to_sys_ptrace(0, MEMORY[0x81A5280], 12, &data);
  if ( buf_len != 32 )
  {
    flag_res = -1;                              // failed
    to_sys_ptrace(0, pid, 13, &data);
    to_sys_ptrace(0, pid, 17, 0);
    to_sys_exit(0);
  }
  
  bignum_init(&internal_buf);
  bignum_init(&r_buf);
  bignum_init(&a2);
  
  convert_from_hex(&const_0, base + 0x12A6, 64);// "d1cc3447d5a9e1e6adae92faaea8770db1fab16b1568ea13c3715f2aeba9d84f"
  convert_from_hex(&const_1, base + 0x1224, 64);// "c10357c7a53fa2f1ef4a5bf03a2d156039e7a57143000c8d8f45985aea41dd31"
  convert_from_hex(&key_part1, base + 0x11E3, 64);// "480022d87d1823880d9e4ef56090b54001d343720dd77cbc5bc5692be948236c"
  convert_from_hex(&const_3, base + 0x11E3, 64);// "480022d87d1823880d9e4ef56090b54001d343720dd77cbc5bc5692be948236c"
  convert_from_hex(&key_part2, base + 0x1265, 64);// "d036c5d4e7eda23afceffbad4e087a48762840ebb18e3d51e4146f48c04697eb"
  
  qmemcpy(&internal_buf, buffer + 0x30, 24u);   // get next part of the password
  
  fd = to_sys_open(0, 0, (char *)(base + 0x11D6));// "/dev/urandom"
  to_sys_read(32u, &r_buf, fd);                 // r_buf = random bytes
  bignum_divmod(&r_buf, &const_0, &div_res, &div_reminder);
  to_sys_close(fd);
  
  bignum_assign(&r_buf, &div_reminder);
  sub_9C3(&const_1, &r_buf, &const_0, &a2);
  
  bignum_assign(&r_buf, &div_reminder);
  sub_9C3(&const_3, &r_buf, &const_0, &out_buf0);
  
  bignum_mul((char *)&internal_buf, (int)&a2, &r_buf); // a2 = "c10357c7a53fa2f1ef4a5bf03a2d156039e7a57143000c8d8f45985aea41dd31"
  bignum_divmod(&r_buf, &const_0, &div_res, &out_buf1);
  
  //print in hex:
  memset(&a4, 0, 0x400u);
  bigint_to_str(&out_buf0, (int)&a4, 0x400);
  memset(&a4, 0, 0x400u);
  bigint_to_str(&out_buf1, (int)&a4, 0x400);
  
  if ( bignum_cmp(&key_part1, &out_buf0) || bignum_cmp(&key_part2, &out_buf1) )// flag verification
                                                // 
  {
    flag_res = -1;                              // failed
    to_sys_ptrace(0, pid, 13, &data);           // PTRACE_SETREGS
    to_sys_ptrace(0, pid, 17, 0);
    to_sys_exit(0);
  }
  buffer[72] = 0;
  write_to_remote_buf(pid, MEMORY[0x81A57C0], (void **)buffer, (signed int)&base, 73);
  flag_res = 32;                                // the flag is ok
  to_sys_ptrace(0, pid, 13, &data);             // PTRACE_SETREGS
  to_sys_ptrace(0, pid, 17, 0);
  to_sys_exit(0);
}

using: https://github.com/kokke/tiny-bignum-c