RSA

base64.c

#include <stdio.h>
#include <string.h>
#include <openssl/bio.h>
#include <openssl/evp.h>
#include <stdint.h>
#include <assert.h>

//B64 encoding credit to https://gist.github.com/barrysteyn/7308212

size_t calcDecodeLength(const char* b64input) { //Calculates the length of a decoded string
	size_t len = strlen(b64input),
		padding = 0;

	if (b64input[len-1] == '=' && b64input[len-2] == '=') //last two chars are =
		padding = 2;
	else if (b64input[len-1] == '=') //last char is =
		padding = 1;

	return (len*3)/4 - padding;
}

int Base64Decode(char* b64message, unsigned char** buffer, size_t* length) { //Decodes a base64 encoded string
	BIO *bio, *b64;

	int decodeLen = calcDecodeLength(b64message);
	*buffer = (unsigned char*)malloc(decodeLen + 1);
	(*buffer)[decodeLen] = '\0';
	bio = BIO_new_mem_buf(b64message, -1);
	b64 = BIO_new(BIO_f_base64());
	bio = BIO_push(b64, bio);

	BIO_set_flags(bio, BIO_FLAGS_BASE64_NO_NL); //Do not use newlines to flush buffer
	*length = BIO_read(bio, *buffer, strlen(b64message));
         //length should equal decodeLen, else something went horribly wrong
	BIO_free_all(bio);

	return (*length == decodeLen); //success
}

rsa.c

#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <unistd.h>

#include <openssl/rsa.h>
#include <openssl/engine.h>
#include <openssl/pem.h>

#include "base64.h"
#include "rsa.h"

int main(){

    // Read flag from file
    char flag[128];
    FILE *flag_f = fopen("flag.txt", "r");
    fgets(flag, 129, flag_f);
    fclose(flag_f);

    // Read user input: base-64 encoded 1024 bit RSA ciphertext
    unsigned char *input = readin();

    // Decrypt user input
    unsigned char plaintext[128];
    decrypt(plaintext, input, "key.pem");
    free(input);

    //Don't let users decrypt the flag!
    if(memcmp(plaintext, flag, 128) == 0){
	    printf("We're not giving away the flag that easy!\n");
    }
    else{
	    write(STDOUT_FILENO, plaintext, 128);
    }
}

void decrypt(unsigned char *out, unsigned char *in, char *prv_file){

    // Read private key from `prv_file`
    FILE *f = fopen(prv_file, "r");
    RSA *privkey = PEM_read_RSAPrivateKey(f, NULL, NULL, NULL);
    fclose(f);

    //Decrypt input
    RSA_private_decrypt(128, in, out, privkey, RSA_NO_PADDING);
}

unsigned char* readin()
{
    // Read up to 180 bytes, terminated by newline
    char buf[180];
    memset(buf, '\0', 180);
    scanf("%179[^\n]s", buf);

    // Decode user input from base64
    unsigned char *cipher_txt;
    size_t len;
    if(!Base64Decode(buf, &cipher_txt, &len)){
	    printf("Input must be base64-encoded");
	    exit(1);
    }
    if(len != 128){
	    printf("RSA ciphertexts must be 128 bytes");
	    exit(1);
    }

    return cipher_txt;
}