Convenient direct memory access, no dealing with endianess or sizes. Seems to compile OK: https://www.godbolt.org/z/acxV6_

RawMemoryAccess.cpp

#include<cstring>
#include<memory>
#include<cstdio>
#include<cstdlib>
#include<limits>
#include<cassert>

template<class T, bool be = false> struct RawMemoryAccess {
	struct Proxy {
		uint8_t *memory;
		Proxy(uint8_t *memory) : memory(memory) {}
		void operator=(T rhs) {
			uintmax_t digits = std::numeric_limits<T>::digits;
			if (be) {
				for (uintmax_t i = 0; i < digits / 8; i++) {
					memory[i] = (rhs >> (digits - ((i + 1) * 8))) & 0xFF;
				}
			} else {
				for (uintmax_t i = 0; i < digits / 8; i++) {
					memory[i] = (rhs >> (i * 8)) & 0xFF;
				}
			}
		}
		operator T() const {
			uintmax_t digits = std::numeric_limits<T>::digits;
			T value = 0;
			if (be) {
				for (uintmax_t i = 0; i < digits / 8; i++) {
					value |= (T)memory[digits / 8 - 1 - i] << (i * 8);
				}
			} else {
				for (uintmax_t i = 0; i < digits / 8; i++) {
					value |= (T)memory[i] << (i * 8);
				}
			}
			return value;
		}
	};
	
	uint8_t *memory;
	RawMemoryAccess(uint8_t *memory) : memory(memory) {}
	Proxy operator[](std::size_t idx) {
		return Proxy(memory + idx);
	}
};

struct Memory {

	std::unique_ptr<uint8_t[]> memory;
	size_t size;
	
	RawMemoryAccess<uint64_t, false> u64;
	RawMemoryAccess<uint32_t, false> u32;
	RawMemoryAccess<uint16_t, false> u16;
	RawMemoryAccess<uint8_t, false> u8;
	
	RawMemoryAccess<uint64_t, true> u64be;
	RawMemoryAccess<uint32_t, true> u32be;
	RawMemoryAccess<uint16_t, true> u16be;
	RawMemoryAccess<uint8_t, true> u8be;
	
	Memory(std::size_t size) :
		memory(std::make_unique<uint8_t[]>(size)),
		size(size),
		u64(memory.get()),
		u32(memory.get()),
		u16(memory.get()),
		u8(memory.get()),
		u64be(memory.get()),
		u32be(memory.get()),
		u16be(memory.get()),
		u8be(memory.get())
	{
	}
};

int main(void) {
	Memory *memory = new Memory(64);
	
	memory->u64[0x00] = 0x1122334455667788ULL;
	memory->u32[0x10] = 0x99AABBCCUL;
	memory->u16[0x0A] = 0xDDEE;
	memory->u8[0x1C] = 0xFF;
	assert(memory->u64[0x00] == 0x1122334455667788ULL);
	assert(memory->u32[0x10] == 0x99AABBCCUL);
	assert(memory->u16[0x0A] == 0xDDEE);
	assert(memory->u8[0x1C] == 0xFF);
	
	memory->u64be[0x20] = 0x1122334455667788ULL;
	memory->u32be[0x30] = 0x99AABBCCUL;
	memory->u16be[0x2A] = 0xDDEE;
	memory->u8be[0x3C] = 0xFF;
	assert(memory->u64be[0x20] == 0x1122334455667788ULL);
	assert(memory->u32be[0x30] == 0x99AABBCCUL);
	assert(memory->u16be[0x2A] == 0xDDEE);
	assert(memory->u8be[0x3C] == 0xFF);
	
	for (size_t i = 0; i < memory->size; i += 16) {
		std::printf("%04zx: ", i);
		for (int j = 0; j < 16 && i + j < memory->size; j++) {
			std::printf("%02X ", (unsigned int)memory->u8[i + j]);
		}
		std::puts("");
	}
	delete memory;
	return 0;
}

/* Output:
0000: 88 77 66 55 44 33 22 11 00 00 EE DD 00 00 00 00
0010: CC BB AA 99 00 00 00 00 00 00 00 00 FF 00 00 00
0020: 11 22 33 44 55 66 77 88 00 00 DD EE 00 00 00 00
0030: 99 AA BB CC 00 00 00 00 00 00 00 00 FF 00 00 00
*/