rsa.c

typedef struct {
    u32 first_byte;
    u32 total_size;
    u32 data_size;
    u32 data_ptr;
} tag_t;

int extract_tags_inside_tag(
    u8* buf, u32 buf_sz, tag_t* tag, tag_t* out_array, u32* num_tags)
{
    if (tag->first_byte != 0x30)
        return 0x800F0024;

    if (buf_sz < tag->total_sz)
        return 0x800F0024;

    u32 prefix_sz = tag->total_size - tag->data_size;

    u8* data_ptr = buf      + prefix_sz;
    u32 data_sz  = buf_size - prefix_sz;

    for (i=0; i<*num_tags && data_sz!=0; i++)
    {
        int rc = extract_tag(data_ptr, data_sz, &out_array[i]);

        if (rc != 0)
            return rc;

        if (data_sz < out_array[i]->total_size)
            return 0x800F0024;

        data_ptr += out_array[i]->total_size;
        data_sz  -= out_array[i]->total_size;
    }

    *num_tags = i;
    return 0;
}

int extract_tag(u8* buf, size_t buf_sz, tag_t* out)
{
    if (buf_sz < 2)
        return 0x800F0016;

    u32 val;

    if (buf[1] & 0x80)
    {
        size_len = buf[1] & 0x7F;

        if (buf_sz < size_len+2)
            return 0x800F0016;

        if (size_len > 4)
            return 0x800F0025;

        out->data_size = 0;

        u32 shift = size_len*8 - 0x18;
        u8* ptr = buf + 1;

        for (i=0; i!=size_len; i++)
        {
            out->data_size |= *(ptr+1) << shift;
            shift -= 8;

            ptr++;
        }
    }
    else
    {
        out->data_size = buf[1];
        size_len = 0;
    }

    out->first_byte = buf[0];
    out->total_size = out->data_size + size_len + 2;
    out->data_ptr   = size_len + 2 + buf;
    return 0;
}

int rsa_verify(u8* buf, u32 buf_sz, u32 hash_type, u8* calculated_hash)
{
    if (buf[0] != 0)
        fail;
    if (buf[1] not in [0, 1])
        fail;

    u8 pad_val;

    if (buf[1] == 0)
        pad_val = 0x00;
    else:
        pad_val = 0xFF;
   
    pos = 2;
   
    for(; pos<10; pos++)
    {
        if (buf[pos] != pad_val)
            fail;
    }
   
    if (rsa_buf_sz < 11)
        loop_cnt = 0;
    else
        loop_cnt = buf_sz - pos;
   
    do {
        if (buf[pos++] != pad_val)
            break;

    } while(loop_cnt--);
   
    if (pad_val == 0xFF) {
        if (buf[pos] != 0)
            fail;
        pos++;
    }

    tag_t first_tag;

    rc = extract_tag(buf + pos, buf_sz - pos, &first_tag);
    if (rc != 0)
        fail;
   
    if (first_tag.first_byte != 0x30)
        fail;

    tag_t tags_within_first_tag[2];
    u32   num_tags = 2;

    rc = extract_tags_inside_tag(
        /* ptr   */ buf + pos,
        /* size  */ buf_sz - pos,
        /* tag   */ &first_tag,
        /* out   */ tags_within_first_tag,
        /* inout */ &num_tags);
    if (rc != 0)
        fail;

    if (num_tags != 2)
        fail;

    if (tags_within_first_tag[0].first_byte != 0x30)
        fail;

    tag_t tags_after_first_tag[2];
    rc = extract_tags_inside_tag(
        /* ptr   */ first_tag->data_ptr,
        /* size  */ tags_out[0]->total_size,
        /* tag   */ &tags_out[0] /* wat */,
        /* out   */ tags_after_first_tag,
        /* inout */ &num_tags);
    if (rc != 0)
        fail;

    if (tags_after_first_tag[0]->first_byte != 6)
        fail;

    padding_t* pad = TABLE[hash_type];

    if (pad->length/*9*/ != tags_after_first_tag[0]->data_size)
        fail;

    if (timingsafe_memcmp(
            tags_after_first_tag[0]->data_ptr,
            pad->data,
            pad->length) != 0)
        fail;

    if (tags_within_first_tag[1].first_byte != 4)
        fail;

    if (tags_within_first_tag[1].data_size != pad->hash_len/*0x20*/)
        fail;
    
    if (timingsafe_memcmp(
            tags_within_first_tag[1]->data_ptr,
            calculated_hash,
            pad->hash_len/*0x20*/) != 0)
        fail;

    return 0;
}