This tool can be used to decrypt PSX package binaries, like main.rel.

PSX_rel.cpp

#include <stdio.h>
#include <stdint.h>
#include <string.h>
#include <openssl/aes.h>
#include <openssl/rsa.h>
#include <algorithm>
#include <openssl/sha.h>

#pragma pack(push, 1)
struct npCoreKey
{
	uint8_t IV[16];
	uint8_t Key[16];
	uint8_t TomCryptVersion;	// 0x91
	uint8_t RSA_Section;		// 0x00 
	uint8_t RSA_SubSection;		// 0x00
	uint8_t padding;			// not used
	uint8_t Type;				// 0x01 (PK_PUBLIC)
	uint32_t ModulusLen;		// 0x101
	uint8_t Modulus[0x101];
	uint32_t ExponentLen;		// 0x3
	uint8_t Exponent[3];
};

struct npCoreSignature
{
	int ContentSize;
	uint8_t SHA1[20];
	uint8_t IV[16];
	uint8_t Key[16];
	int Flags;
};

struct npCoreHeader
{
	npCoreSignature PlaintextSignature;

	uint8_t TomCryptVersion;	// 0x91
	uint8_t RSA_Section;		// 0x00 
	uint8_t RSA_SubSection;		// 0x02
	uint8_t padding;			// not used
	uint32_t RSASignatureLen;	// 0x100
	uint8_t RSASignature[0x100];
};
#pragma pack(pop)

//------------------------------------------------------------
//-----------       Created with 010 Editor        -----------
//------         www.sweetscape.com/010editor/          ------
//
// File    : C:\Users\samicrusader\Desktop\osd\xosdmain_unp.elf
// Address : 18800 (0x4970)
// Size    : 305 (0x131)
//------------------------------------------------------------
uint8_t KeyBlob[305] = {
    0x77, 0xDE, 0x01, 0x98, 0xD1, 0x40, 0x4C, 0xD9, 0xF2, 0xFE, 0xA5, 0xAC, 0xA0, 0x94, 0xC7, 0x96,
    0xEB, 0x66, 0x2A, 0x59, 0x63, 0x45, 0x9C, 0x48, 0x84, 0x77, 0x63, 0x63, 0x23, 0x97, 0x66, 0x13,
    0x91, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x00, 0x00, 0x02, 0x5F, 0x78, 0x33, 0xFB, 0xF3, 0xA1,
    0x45, 0x1E, 0x19, 0x3E, 0xD6, 0x0A, 0x50, 0x9B, 0x47, 0x87, 0x9C, 0x2D, 0xAE, 0xD7, 0xAC, 0x59,
    0x33, 0x8D, 0x47, 0x8C, 0x5F, 0x4A, 0x97, 0x86, 0x3B, 0x8D, 0xD7, 0x8B, 0xFB, 0x12, 0xDE, 0x8B,
    0xBB, 0xF3, 0x72, 0x4D, 0x8C, 0x58, 0xFC, 0x30, 0x05, 0xC8, 0xA8, 0x21, 0x1D, 0x3A, 0xF0, 0x17,
    0x83, 0xF2, 0x04, 0xE1, 0x08, 0x7F, 0xD9, 0x4D, 0xAE, 0x84, 0xDD, 0x19, 0x95, 0x2F, 0xB9, 0x5F,
    0xF8, 0x58, 0x14, 0x6E, 0x38, 0xE8, 0xB2, 0x3B, 0x64, 0x9A, 0xA5, 0xC6, 0x6A, 0x4D, 0x39, 0xA6,
    0x06, 0xDB, 0x87, 0x93, 0x6D, 0x04, 0x1A, 0x26, 0xA0, 0xEE, 0x79, 0xD8, 0x6C, 0xD4, 0x3E, 0xD9,
    0xA9, 0x89, 0x78, 0x78, 0x17, 0x85, 0x41, 0x3D, 0x9D, 0x17, 0x3C, 0x85, 0x1E, 0xBC, 0xDF, 0xA7,
    0x93, 0x43, 0x7B, 0x49, 0xA5, 0x47, 0xCE, 0xE4, 0x6C, 0x62, 0x62, 0x92, 0x34, 0xB5, 0x85, 0xCD,
    0xE9, 0xD1, 0x55, 0x7B, 0x96, 0x9D, 0xC1, 0x50, 0x40, 0x30, 0x42, 0xE6, 0x51, 0x41, 0x56, 0xDA,
    0xA6, 0xF3, 0x32, 0x9F, 0xDA, 0xC2, 0x1E, 0xB6, 0xB1, 0xD1, 0xBE, 0x39, 0xF9, 0x97, 0x1C, 0xE8,
    0x60, 0xC8, 0xFA, 0xE4, 0x2E, 0x9B, 0x93, 0xD5, 0x74, 0xC8, 0x72, 0x28, 0xF0, 0xE6, 0x73, 0x76,
    0x6E, 0x2E, 0x17, 0x0C, 0x5D, 0x23, 0x18, 0x17, 0xAF, 0x62, 0xD9, 0x05, 0x39, 0xB0, 0xAC, 0xA9,
    0x54, 0xEB, 0x31, 0xB5, 0xCA, 0x08, 0xD7, 0x4F, 0x6F, 0xFA, 0x9B, 0xF5, 0x0A, 0xC3, 0x4C, 0xF5,
    0x55, 0x6A, 0xC4, 0x92, 0x5A, 0x2C, 0x64, 0x34, 0x6F, 0xA0, 0xC6, 0x2B, 0x17, 0x66, 0x40, 0x87,
    0xCE, 0xBC, 0x32, 0xB7, 0xF1, 0x51, 0x73, 0x77, 0xF9, 0x87, 0x5F, 0x07, 0xAF, 0x19, 0xDD, 0xD9,
    0xB9, 0xB4, 0x54, 0x36, 0xB7, 0xEC, 0x90, 0x36, 0x5E, 0xEF, 0x03, 0x00, 0x00, 0x00, 0x01, 0x00,
    0x01 
};


// What the fuck is that?
int rsa_signdepad(const uint8_t* in, uint32_t inlen, uint8_t* out, uint32_t* outlen)
{
	if (*outlen < inlen / 3) {
		return -1;
	}

	/* check padding bytes */
	for (unsigned long  x = 0; x < inlen / 3; x++) {
		if (in[x] != (unsigned char)0xFF || in[x + (inlen / 3) + (inlen / 3)] != (unsigned char)0xFF) {
			return -1;
		}
	}
	for (unsigned long x = 0; x < inlen / 3; x++)
		out[x] = in[x + (inlen / 3)];
	*outlen = inlen / 3;
	return 0;
}

int main()
{
	npCoreKey* key = (npCoreKey*)KeyBlob;

	FILE* f = fopen(argv[1], "rb");
	fseek(f, 0, SEEK_END);
	int len = ftell(f) - 0x400;
	fseek(f, 0, SEEK_SET);
	uint8_t encryptedHeader[0x400];
	fread(encryptedHeader, 1, 0x400, f);
	uint8_t* data = new uint8_t[len];
	fread(data, 1, len, f);
	fclose(f);

	AES_KEY skey;
	AES_set_decrypt_key(key->Key, 16 * 8, &skey);

	AES_cbc_encrypt(encryptedHeader, encryptedHeader, 0x400, &skey, key->IV, AES_DECRYPT);

	npCoreHeader* header = (npCoreHeader*)encryptedHeader;

	RSA* rsa = RSA_new();

	BIGNUM* Modulus = BN_new();
	BN_bin2bn(key->Modulus, key->ModulusLen, Modulus);
	BIGNUM* Exponent = BN_new();
	BN_bin2bn(key->Exponent, key->ExponentLen, Exponent);
	RSA_set0_key(rsa, Modulus, Exponent, NULL);

	uint8_t Decrypted[0x101];
	int rsalen = RSA_public_decrypt(header->RSASignatureLen, header->RSASignature, Decrypted, rsa, RSA_NO_PADDING);

	RSA_free(rsa);

	// what the fuck? why?!
	uint8_t* RsaContent = Decrypted;
	uint32_t RsaContentLen = rsalen;
	while (*(++RsaContent) == 0)
		RsaContentLen--;

	uint8_t Depadded[0x101];
	uint32_t outlen = 0x101;
	if (rsalen != key->ModulusLen || rsa_signdepad(RsaContent--, RsaContentLen, Depadded, &outlen) != 0 || outlen != sizeof(npCoreSignature) || memcmp(header, Depadded, sizeof(npCoreSignature)) != 0)
	{
		printf("Invalid RSA signature!\n");
		return -1;
	}

	if ((header->PlaintextSignature.Flags & 1) == 0)
	{
		AES_KEY dkey;
		AES_set_decrypt_key(header->PlaintextSignature.Key, 16 * 8, &dkey);
		AES_cbc_encrypt(data, data, len, &dkey, header->PlaintextSignature.IV, AES_DECRYPT);
	}

	if (memcmp(SHA1(data, len, NULL), header->PlaintextSignature.SHA1, 20) != 0)
	{
		printf("Invalid hash signature!\n");
		return -1;
	}
	
	FILE *fo = fopen(argv[2], "wb");
	fwrite(data, 1, len, fo);
	fclose(fo);

	delete [] data;

	return 0;
}