Custom syscall for extended symbol resolving (allow specifying of library name, flags to be able to use mangled symbol names) on PS4.

sys_dynlib_dlsym_ex.c

//
// Custom syscall for extended symbol resolving (allow specifying of library name, flags to be able to use mangled symbol names) on PS4.
//
// NOTE: slide offsets are for 5.00/5.01 kernel.
//

//
// PATCHES (syntax: offset,name,old,new).
//

// Allow sys_dynlib_dlsym in all processes.
DECLARE_PATCH_64(0x237E2A, allow_dynlib_dlsym, H64S(0F,82,C0,01,00,00,48,8B), H64S(E9,C1,01,00,00,90,48,8B));

// Don't restrict dynlib information.
DECLARE_PATCH_32(0x2B2350, dont_restrict_dynlib_info, H32S(65,48,8B,04), H32S(31,C0,C3,90));

// Allow usage of mangled symbols in dynlib_do_dlsym().
DECLARE_PATCH_BUF(0x2AF877, allow_mangled_symbols_in_dlsym, BYTES(45,31,FF,45,31,F6), BYTES(90,90,90,90,90,90));

//
// Kernel symbols, structures and definitions.
//

#define DYNLIB_DLSYM_FLAG_MANGLED_NAME 0x1

struct dynlib_obj;

#define SIZEOF_DYNLIB 0x100 // dynlib_proc_initialize_step1

TYPE_BEGIN(struct dynlib, SIZEOF_DYNLIB);
	TYPE_FIELD(SLIST_HEAD(, dynlib_obj) objs, 0x00);
	// ...
	TYPE_FIELD(struct dynlib_obj* main_obj, 0x10);
	// ...
	TYPE_FIELD(struct sx bind_lock, 0x70);
	// ...
TYPE_END();

#if FW_VER == 501
	DECLARE_FUNCTION(0x2AF7B0, dynlib_do_dlsym, void*, struct dynlib* dl, struct dynlib_obj* obj, char* name, char* libname, unsigned int flags);
	DECLARE_FUNCTION(0x2B0B40, dynlib_find_obj_by_handle, struct dynlib_obj*, struct dynlib* dl, int handle);
#endif

//
// Kernel code for payload.
//

enum {
	// ...
	SUPERCALL_DLSYM,
	// ...
};

struct sys_supercall_args {
	int cmd;
	int cmd__pad;

	union {
		// ...

#if FW_VER == 501
		struct {
			int handle;
			int handle__pad;
			char* symbol;
			char* lib;
			unsigned int flags;
			void** addr;
		} dlsym;
#endif
		
		// ...
	};
};

int sys_supercall(struct thread* td, struct sys_supercall_args* uap) {
	int ret = 0;

	td->td_retval[0] = 0;

	switch (uap->cmd) {
		// ...
			
	#if FW_VER == 501
		case SUPERCALL_DLSYM: {
			struct proc* proc = td->td_proc;
			struct dynlib* dynlib;
			struct dynlib_obj* main_obj;
			struct dynlib_obj* obj;
			char symbol[0xA00];
			char lib[0xA00];
			void* addr = NULL;
			size_t n;

			ret = copyinstr(uap->dlsym.symbol, symbol, sizeof(lib), &n);
			if (ret)
				goto err;
			ret = copyinstr(uap->dlsym.lib, lib, sizeof(lib), &n);
			if (ret)
				goto err;

			dynlib = proc->p_dynlib;
			if (!dynlib) {
err_not_dll:
				RTLD_ERR("this is not dynamic linked program.\n");
				ret = EPERM;
				goto err;
			}

			sx_xlock(&dynlib->bind_lock);
			{
				main_obj = dynlib->main_obj;
				if (!main_obj) {
					sx_xunlock(&dynlib->bind_lock);
					goto err_not_dll;
				}

				obj = dynlib_find_obj_by_handle(dynlib, uap->dlsym.handle);
				if (!obj) {
					sx_xunlock(&dynlib->bind_lock);
					ret = ESRCH;
					goto err;
				}

				addr = dynlib_do_dlsym(dynlib, obj, symbol, (*lib != '\0') ? lib : NULL, uap->dlsym.flags);
				if (!addr) {
					sx_xunlock(&dynlib->bind_lock);
					ret = ESRCH;
					goto err;
				}
			}
			sx_xunlock(&dynlib->bind_lock);

			ret = copyout(&addr, (void*)uap->dlsym.addr, sizeof(addr));
			break;
		}
#endif

		default:
			ret = EINVAL;
			goto err;
	}

err:
	return ret;
}

// ...

#define SYS_mac_syscall 394

static void setup_custom_syscalls(void) {
	// ...
	set_syscall_handler(SYS_mac_syscall, &sys_supercall, 8);
	// ...
}

//
// User land code.
//

enum {
	// ...
	SUPERCALL_DLSYM,
	// ...
};

#define SYS_supercall 394
// ...
#define SYS_dynlib_dlsym 591
#define SYS_dynlib_load_prx 594
#define SYS_dynlib_get_info_ex 608

// ...

typedef int module_id_t;

#define MAX_MODULE_NAME_LENGTH 256
#define MAX_MODULE_SEGMENTS 4

#define FINGERPRINT_SIZE 20

struct module_segment {
	uint64_t addr;
	uint32_t size;
	uint32_t flags;
};

struct module_info_ex {
	size_t st_size;
	char name[MAX_MODULE_NAME_LENGTH];
	module_id_t id;
	uint32_t tls_index;
	uint64_t tls_init_addr;
	uint32_t tls_init_size;
	uint32_t tls_size;
	uint32_t tls_offset;
	uint32_t tls_align;
	uint64_t init_proc_addr;
	uint64_t fini_proc_addr;
	uint64_t reserved1;
	uint64_t reserved2;
	uint64_t eh_frame_hdr_addr;
	uint64_t eh_frame_addr;
	uint32_t eh_frame_hdr_size;
	uint32_t eh_frame_size;
	struct module_segment segments[MAX_MODULE_SEGMENTS];
	uint32_t segment_count;
	uint32_t ref_count;
};

// ...

static inline int load_module(const char* file_path, module_id_t* id) {
	int ret = SYSCALL(SYS_dynlib_load_prx, file_path, 0 /* reserved */, id);
	return ret;
}

static inline int get_module_info_ex(module_id_t id, struct module_info_ex* info) {
	int ret;

	memset(info, 0, sizeof(*info));
	info->st_size = sizeof(*info);

	ret = SYSCALL(SYS_dynlib_get_info_ex, id, 0 /* reserved */, info);

	return ret;
}

static inline int start_module(module_id_t id, void* args, size_t args_size) {
	struct module_info_ex info;
	int (*module_start)(size_t args_size, const void* args);
	int ret;

	ret = get_module_info_ex(id, &info);
	if (ret != 0)
		return ret;

	if (info.ref_count < 2) {
		module_start = (void*)info.init_proc_addr;
		if (!module_start)
			return -1;
		return module_start(args_size, args);
	} else {
		return 0;
	}
}

static inline void* lookup_module_symbol(module_id_t id, const char* name) {
	void* addr = NULL;
	int ret;

	ret = SYSCALL(SYS_dynlib_dlsym, id, name, &addr);
	if (ret < 0)
		goto error;

error:
	return addr;
}

#if FW_VER == 501
	static inline void* lookup_module_symbol_ex(module_id_t id, const char* name, const char* lib, unsigned int flags) {
		void* addr = NULL;
		int ret;

		ret = SYSCALL(SYS_supercall, SUPERCALL_DLSYM, id, name, lib, flags, &addr);
		if (ret < 0)
			goto error;

	error:
		return addr;
	}
#endif

// ...

module_id_t mid = 0x2001; // libkernel

void* addr1 = (void*)lookup_module_symbol(mid, "sceKernelOpen");
void* addr2 = (void*)lookup_module_symbol_ex(mid, "1G3lF1Gg1k8", "libkernel", DYNLIB_DLSYM_FLAG_MANGLED_NAME);

dprintf("%p %p", addr1, addr2);