Phobos - RSA idenfitication

decompiled.cpp

//decompiled functions (HexRays), manually processed

void __cdecl RSA_pub_key_new(_RSA_CTX *ctx, int a2, int a3, int a4, int a5)
{
  struct _bigint *v5; // eax
  struct _bigint *v6; // ebx
  _DWORD *v7; // eax
  _bigint *v8; // edi
  int v9; // esi
  unsigned int v10; // ecx
  int v11; // eax
  int v12; // eax
  int num; // [esp+14h] [ebp+8h]

  if ( ctx->m )
    RSA_free((_RSA_CTX *)ctx->m);
  v5 = (struct _bigint *)calloc(1u, 0x28u);
  v6 = v5;
  v7 = int_to_bi(2, v5);
  v6->refs = (int)v7;
  *(_WORD *)v7[3] = 0;
  *(_WORD *)(*(_DWORD *)(v6->refs + 12) + 2) = 1;
  bi_permanent(v6->refs);
  v8 = (_bigint *)calloc(1u, 0x14u);
  ctx->m = v8;
  v8[1].next = v6;
  v8->comps = (_DWORD *)a3;
  v9 = bi_import(a3, v6, a2);
  v8->next = (struct _bigint *)v9;
  v10 = *(unsigned __int16 *)(*(_DWORD *)(v9 + 12) + 2 * *(signed __int16 *)(v9 + 4) - 2) + 1;
  v6->comps = (_DWORD *)v9;
  num = (unsigned __int16)(0x10000 / v10);
  bi_permanent(v9);
  bi_int_multiply(v6, v9, num);
  v6[1].next = (struct _bigint *)v11;
  bi_permanent(v11);
  v12 = bi_import(a5, v6, a4);
  *(_DWORD *)&v8->size = v12;
  bi_permanent(v12);
}

size_t __usercall RSA_encrypt@<eax>(int out_data@<eax>, _RSA_CTX *ctx@<ecx>, void *in_data, __int16 in_len)
{
  BYTE *out_data_ptr; // esi
  _RSA_CTX *_ctx; // edi
  signed int num_pads_needed; // ebx
  int v7; // eax
  BYTE weak_random_val; // al
  _BYTE *_out_data_ptr2; // ebx
  _bigint *dat_bi; // eax
  _bigint *encrypt_bi; // eax
  size_t byte_size; // [esp+10h] [ebp-8h]
  signed int i; // [esp+14h] [ebp-4h]

  out_data_ptr = (BYTE *)out_data;
  _ctx = ctx;
  byte_size = ctx->num_octets;
  num_pads_needed = byte_size - (unsigned __int16)in_len - 3;
  *(_WORD *)out_data = 0x200;
  if ( (phProv || CryptAcquireContextW(&phProv, 0, 0, 0x18u, 0xF0000000))
    && CryptGenRandom(phProv, num_pads_needed, out_data_ptr + 2) )// get_random_NZ
  {
    for ( i = 0; i < num_pads_needed; ++i )
    {
      if ( !out_data_ptr[i + 2] )
      {
        do
        {
          weak_random_val = rand();
          out_data_ptr[i + 2] = weak_random_val;
        }
        while ( !weak_random_val );
      }
    }
    v7 = byte_size - (unsigned __int16)in_len - 3;
  }
  else
  {
    v7 = 0;
  }
  if ( (v7 == 0) < 0 )
    return -1;
  _out_data_ptr2 = &out_data_ptr[num_pads_needed];
  _out_data_ptr2[2] = 0;
  memcpy_0(_out_data_ptr2 + 3, in_data, (unsigned __int16)in_len);
  dat_bi = (_bigint *)bi_import(byte_size, &_ctx->bi_ctx->active_list, (int)out_data_ptr);
  _ctx->bi_ctx->mod_offset = 0;
  bi_mod_power(_ctx->bi_ctx, dat_bi, (int)_ctx->e);
  bi_export(byte_size, encrypt_bi, _ctx->bi_ctx, out_data_ptr);
  bi_terminate((int)_ctx->bi_ctx);
  return byte_size;
}

void __usercall RSA_free(_RSA_CTX *rsa_ctx@<ebx>)
{
  _BI_CTX *bi_ctx; // edi

  if ( rsa_ctx )
  {
    bi_ctx = rsa_ctx->bi_ctx;
    bi_depermanent(rsa_ctx->e);
    bi_free(bi_ctx, rsa_ctx->e);
  }
}

void __usercall bi_mod_power(_BI_CTX *ctx@<eax>, _bigint *bi, int a3)
{
  _BI_CTX *_ctx; // ebx
  int v4; // eax
  signed int v5; // edx
  signed int v6; // esi
  _DWORD *v7; // eax
  int v8; // edi
  struct _bigint *v9; // ST28_4
  int *ctx_g; // eax
  int v11; // esi
  int v12; // eax
  int v13; // eax
  int v14; // eax
  int v15; // eax
  int v16; // eax
  int v17; // eax
  unsigned __int16 v18; // [esp+Ch] [ebp-8h]
  int v19; // [esp+Ch] [ebp-8h]
  int v20; // [esp+10h] [ebp-4h]

  _ctx = ctx;
  v4 = *(signed __int16 *)(a3 + 4);
  v5 = 15;
  v18 = 0x8000u;
  while ( !(v18 & *(_WORD *)(*(_DWORD *)(a3 + 12) + 2 * v4 - 2)) )
  {
    v18 >>= 1;
    v6 = v5--;
    if ( !v6 )
    {
      v20 = -1;
LABEL_5:
      v7 = int_to_bi(1, _ctx);
      *(_WORD *)v7[3] = 1;
      v8 = (int)v7;
      _ctx->g = (struct _bigint **)malloc(4u);
      v9 = (struct _bigint *)int_to_bi(bi->size, _ctx);
      memcpy_0(v9->comps, bi->comps, 2 * bi->size);
      *_ctx->g = v9;
      ctx_g = (int *)_ctx->g;
      _ctx->window = 1;
      bi_permanent(*ctx_g);
      while ( 1 )
      {
        if ( exp_bit_is_one(v20) )
        {
          v11 = v20;
          v19 = 0;
          if ( v20 >= 0 )
          {
            while ( !exp_bit_is_one(v11) )
              ++v11;
          }
          else
          {
            v11 = 0;
          }
          for ( ; v20 >= v11; --v20 )
          {
            v12 = *(_DWORD *)(v8 + 8);
            if ( v12 != 0x7FFF55AA )
              *(_DWORD *)(v8 + 8) = v12 + 1;
            sub_4048CD(_ctx, v8, v8);
            v8 = bi_residue((int)_ctx, v13, (int)_ctx->bi_mod[(unsigned __int8)_ctx->mod_offset]);
            if ( exp_bit_is_one(v20) )
              ++v19;
            if ( v20 != v11 )
              v19 *= 2;
          }
          sub_4048CD(_ctx, v8, (int)_ctx->g[(v19 - 1) / 2]);
          v15 = bi_residue((int)_ctx, v14, (int)_ctx->bi_mod[(unsigned __int8)_ctx->mod_offset]);
          v20 = v11 - 1;
        }
        else
        {
          v16 = *(_DWORD *)(v8 + 8);
          if ( v16 != 0x7FFF55AA )
            *(_DWORD *)(v8 + 8) = v16 + 1;
          sub_4048CD(_ctx, v8, v8);
          v15 = bi_residue((int)_ctx, v17, (int)_ctx->bi_mod[(unsigned __int8)_ctx->mod_offset]);
          --v20;
        }
        v8 = v15;
        if ( v20 < 0 )
        {
          if ( _ctx->window > 0 )
          {
            bi_depermanent(*_ctx->g);
            bi_free(_ctx, *_ctx->g);
          }
          free(_ctx->g);
          bi_free(_ctx, bi);
        }
      }
    }
  }
  v20 = v5 + 16 * v4 - 16;
  goto LABEL_5;
}

void __usercall bi_export(size_t size@<eax>, _bigint *x@<esi>, _BI_CTX *ctx, void *data)
{
  signed int k; // edi
  signed int v5; // eax
  char v6; // [esp+4h] [ebp-8h]
  signed int v7; // [esp+8h] [ebp-4h]

  k = size - 1;
  memset(data, 0, size);
  v5 = 0;
  if ( x->size > 0 )
  {
    do
    {
      v6 = 0;
      v7 = 0;
      do
      {
        *((_BYTE *)data + k--) = (unsigned __int16)((255 << v7) & *((_WORD *)x->comps + v5)) >> 8 * v6;
        if ( k < 0 )
          goto LABEL_6;
        v7 += 8;
        ++v6;
      }
      while ( v7 < 16 );
      ++v5;
    }
    while ( v5 < x->size );
  }
LABEL_6:
  bi_free(ctx, x);
}

void __usercall bi_terminate(_BI_CTX *a1@<edi>)
{
  struct _bigint *v1; // esi
  struct _bigint *v2; // ebx

  v1 = a1->free_list;
  if ( v1 )
  {
    do
    {
      v2 = v1->next;
      free(v1->comps);
      free(v1);
      v1 = v2;
    }
    while ( v2 );
    a1->free_count = 0;
    a1->free_list = 0;
  }
}

_BI_CTX *__usercall __noreturn bi_free@<eax>(_BI_CTX *ctx@<eax>, _bigint *bi@<ecx>)
{
  int bi_refs; // edx
  int _bi_refs; // edx
  bool v4; // sf

  bi_refs = bi->refs;
  if ( bi_refs != 0x7FFF55AA )
  {
    _bi_refs = bi_refs - 1;
    bi->refs = _bi_refs;
    if ( _bi_refs <= 0 )
    {
      bi->next = ctx->free_list;
      ++ctx->free_count;
      v4 = ctx->active_count-- - 1 < 0;
      ctx->free_list = bi;
      if ( v4 )
        abort();
    }
  }
  return ctx;
}

functions.csv

structs.h

//identified structs

struct _bigint
{
    struct _bigint* next;
    short size;
    short max_comps;
    int refs;
    _DWORD* comps;
};

struct _BI_CTX
{
    struct _bigint *active_list;                    /**< Bigints currently used. */
    struct _bigint *free_list;                      /**< Bigints not used. */
    struct _bigint *bi_radix;                       /**< The radix used. */
    struct _bigint *bi_mod[1];        /**< modulus */
    struct _bigint *bi_normalised_mod[1]; /**< Normalised mod storage. */
    struct _bigint **g;                 /**< Used by sliding-window. */
    int window;                 /**< The size of the sliding window */
    int active_count;           /**< Number of active bigints. */
    int free_count;             /**< Number of free bigints. */
    _BYTE mod_offset;         /**< The mod offset we are using */
};

struct _RSA_CTX
{
    _bigint *m;              /* modulus */
    _bigint *e;              /* public exponent */
    _bigint *d;              /* private exponent */
    _DWORD num_octets;
    struct _BI_CTX *bi_ctx;
};

https://github.com/joyent/syslinux/tree/master/gpxe/src/crypto/axtls