This was just a homework so it needs few improvements for real use.

ascii85.c

// ascii85 encoder/decoder prototype

#include <stdio.h>
#include <stdlib.h>
#include <string.h>

int ascii85encode(unsigned char *input, char *output)
{
        size_t inputSize = strlen((char *)input), inputIndex = 0;
	unsigned int i = 0, j = 0, chunk = 0;

	// In case string length is not divisible by 4 we need to use \0
	if (inputSize % 4 != 0) {
		for (i = 0; i < 4 - (inputSize % 4); i++) {
			input[i + inputSize] = '\0';
		}

		inputSize += 4 - (inputSize % 4);
	}
	// Encode
	for (i = 0; inputIndex < inputSize - 1; i += 5, inputIndex += 4) {
		// Pull out one 32 bit long number, byte by byte
		chunk = 0;
		for (j = 0; j < 4; j++) {
			if (inputIndex + j < inputSize) {
				chunk |= input[inputIndex + j] << 8 * (3 - j);
			}
		}

		// Divison remainders
		for (j = 5; j > 0; j--) {	// 5 to 1 and j - 1 because unsigned int cant reach -1
			output[i + j - 1] = chunk % 85 + 33;
			chunk /= 85;
		}
	}
	output[i] = '\0';

	return 1;
}

int ascii85decode(unsigned char *input, char *output, size_t * outputSize)
{
	size_t inputSize = strlen((char *)input), outputIndex = 0;
	unsigned int i = 0, j = 0, chunk = 0;

	for (i = 0; i < inputSize; i += 5, outputIndex += 4) {

		// Reconstruct original 32 bit number out of 5 bytes
		chunk = (input[i] - 33) * 85;
		for (j = 1; j < 4; j++) {
			chunk = (chunk + input[i + j] - 33) * 85;
		}
		chunk += input[i + 4] - 33;

		// Pull out decoded bytes one by one
		for (j = 0; j < 4; j++) {
			output[outputIndex + j] = chunk >> 8 * (3 - j);
		}
	}

	*outputSize = outputIndex;

	return 1;
}

int main(int argc, char **argv)
{
	size_t i = 0;
	long int buffer;

	if (argc == 2 && !strcmp(argv[1], "-d")) {
		/* Decoding mode */
		unsigned char input[50];
		char output[40];

		for (; (i < 50) && (buffer = getchar()) && (buffer != '\n')
		     && (buffer != EOF); input[i] = buffer, i++) ;
		input[i] = '\0';

		size_t outputSize;
		ascii85decode(input, output, &outputSize);

		for (i = 0; i <= outputSize; i++) {
			printf("%c", output[i]);
		}

	} else {
		/* Encoding mode */
		unsigned char input[40];
		char output[50];

		for (; (i < 40) && (buffer = getchar()) && (buffer != '\n')
		     && (buffer != EOF); input[i] = buffer, i++) ;
		input[i] = '\0';

		ascii85encode(input, output);
		printf("%s\n", output);
	}

	return 0;
}