This tool can be used to decrypt PSX package binaries, like main.rel. (You need to bring your own key!)

PSX_rel.cpp

#include <stdio.h>
#include <stdint.h>
#include <string.h>
#include <openssl/aes.h>
#include <openssl/rsa.h>
#include <algorithm>
#include <openssl/sha.h>

#pragma pack(push, 1)
struct npCoreKey
{
	uint8_t IV[16];
	uint8_t Key[16];
	uint8_t TomCryptVersion;	// 0x91
	uint8_t RSA_Section;		// 0x00 
	uint8_t RSA_SubSection;		// 0x00
	uint8_t padding;			// not used
	uint8_t Type;				// 0x01 (PK_PUBLIC)
	uint32_t ModulusLen;		// 0x101
	uint8_t Modulus[0x101];
	uint32_t ExponentLen;		// 0x3
	uint8_t Exponent[3];
};

struct npCoreSignature
{
	int ContentSize;
	uint8_t SHA1[20];
	uint8_t IV[16];
	uint8_t Key[16];
	int Flags;
};

struct npCoreHeader
{
	npCoreSignature PlaintextSignature;

	uint8_t TomCryptVersion;	// 0x91
	uint8_t RSA_Section;		// 0x00 
	uint8_t RSA_SubSection;		// 0x02
	uint8_t padding;			// not used
	uint32_t RSASignatureLen;	// 0x100
	uint8_t RSASignature[0x100];
};
#pragma pack(pop)

uint8_t KeyBlob[305] = /*
 * Can be found in the xosdmain.elf on xfrom or hdd (use kelftool to decrypt that)
 * SHA256: d3c6954bb6c8c122ba547f4294c90ba739e5bfa8df37a293f532bab16a404e45
 */;

// What the fuck is that?
int rsa_signdepad(const uint8_t* in, uint32_t inlen, uint8_t* out, uint32_t* outlen)
{
	if (*outlen < inlen / 3) {
		return -1;
	}

	/* check padding bytes */
	for (unsigned long  x = 0; x < inlen / 3; x++) {
		if (in[x] != (unsigned char)0xFF || in[x + (inlen / 3) + (inlen / 3)] != (unsigned char)0xFF) {
			return -1;
		}
	}
	for (unsigned long x = 0; x < inlen / 3; x++)
		out[x] = in[x + (inlen / 3)];
	*outlen = inlen / 3;
	return 0;
}

int main()
{
	npCoreKey* key = (npCoreKey*)KeyBlob;

	FILE* f = fopen(argv[1], "rb");
	fseek(f, 0, SEEK_END);
	int len = ftell(f) - 0x400;
	fseek(f, 0, SEEK_SET);
	uint8_t encryptedHeader[0x400];
	fread(encryptedHeader, 1, 0x400, f);
	uint8_t* data = new uint8_t[len];
	fread(data, 1, len, f);
	fclose(f);

	AES_KEY skey;
	AES_set_decrypt_key(key->Key, 16 * 8, &skey);

	AES_cbc_encrypt(encryptedHeader, encryptedHeader, 0x400, &skey, key->IV, AES_DECRYPT);

	npCoreHeader* header = (npCoreHeader*)encryptedHeader;

	RSA* rsa = RSA_new();

	BIGNUM* Modulus = BN_new();
	BN_bin2bn(key->Modulus, key->ModulusLen, Modulus);
	BIGNUM* Exponent = BN_new();
	BN_bin2bn(key->Exponent, key->ExponentLen, Exponent);
	RSA_set0_key(rsa, Modulus, Exponent, NULL);

	uint8_t Decrypted[0x101];
	int rsalen = RSA_public_decrypt(header->RSASignatureLen, header->RSASignature, Decrypted, rsa, RSA_NO_PADDING);

	RSA_free(rsa);

	// what the fuck? why?!
	uint8_t* RsaContent = Decrypted;
	uint32_t RsaContentLen = rsalen;
	while (*(++RsaContent) == 0)
		RsaContentLen--;

	uint8_t Depadded[0x101];
	uint32_t outlen = 0x101;
	if (rsalen != key->ModulusLen || rsa_signdepad(RsaContent--, RsaContentLen, Depadded, &outlen) != 0 || outlen != sizeof(npCoreSignature) || memcmp(header, Depadded, sizeof(npCoreSignature)) != 0)
	{
		printf("Invalid RSA signature!\n");
		return -1;
	}

	if ((header->PlaintextSignature.Flags & 1) == 0)
	{
		AES_KEY dkey;
		AES_set_decrypt_key(header->PlaintextSignature.Key, 16 * 8, &dkey);
		AES_cbc_encrypt(data, data, len, &dkey, header->PlaintextSignature.IV, AES_DECRYPT);
	}

	if (memcmp(SHA1(data, len, NULL), header->PlaintextSignature.SHA1, 20) != 0)
	{
		printf("Invalid hash signature!\n");
		return -1;
	}
	
	FILE *fo = fopen(argv[2], "wb");
	fwrite(data, 1, len, fo);
	fclose(fo);

	delete [] data;

	return 0;
}