Object.setPrototypeOf

example.js

var log = console.log.bind(console);

function CustomError(message) {
	//...
}

//By default, the prototype of all new functions is Function.prototype
log(Object.getPrototypeOf(CustomError) === Function.prototype); //true

//Setting the prototype of CustomError constructor using Object.setPrototypeOf()
//changes the prototype of the constructor function itself, not new instances created by it.
Object.setPrototypeOf(CustomError, Error.prototype);

var e = new CustomError();

log( Object.getPrototypeOf(e) === Error.prototype ) //false
log( Object.getPrototypeOf(e) === CustomError.prototype ) //true
log( e instanceof CustomError ) //true
//this should be true; it's false because it's not actually inheriting from Error
log( e instanceof Error ) //false

//The only thing that changed was the prototype of the constructor itself
log( Object.getPrototypeOf(CustomError) )

//The prototype property on constructor functions is not the same as the actual prototype of the object.
//(That's why the Object.setPrototypeOf() approach doesn't work correctly.)
log( Object.getPrototypeOf(CustomError) === CustomError.prototype ) //false
log( Object.getPrototypeOf(CustomError) === CustomError.__proto__ ) //true
log( Object.getPrototypeOf(CustomError) === Object.getPrototypeOf(e) ) //false


//THE CORRECT WAY

function CustomError2(message) {
	//...
}

//In order to affect the prototype of *instances* of CustomError, you can simply
//set the prototype property of the constructor
CustomError2.prototype = Object.create(Error.prototype);

var e2 = new CustomError2();
log( Object.getPrototypeOf(e2) === CustomError2.prototype ) //true

//Now instanceof works correctly
log( e2 instanceof CustomError2 ) //true
log( e2 instanceof Error ) //true

//You *could* use Object.setPrototypeOf() to *change* the prototype of an already-existing
//error object. That's really the only reason you should ever need to use Object.setPrototypeOf()
// - if you already have an object and want to change its prototype. But this is less efficient
//and usually unnecessarily complicated compared with setting the prototype to what you want it
//to be in the first place.

function CustomError3(message) {}
var e3 = new CustomError3();
Object.setPrototypeOf(e3, Error.prototype);

log( Object.getPrototypeOf(e3) === CustomError3.prototype ) //false
log( Object.getPrototypeOf(e3) === Error.prototype ) //true

//There *is* one thing you can do using this technique that you can't do without Object.setPrototypeOf():
//you can make a custom error look more like a regular Error when it's shown in the console and still
//have it be an instance of your custom error type.

function CustomError4(message) {
	var err = new Error(message);
	Object.setPrototypeOf(err, CustomError4.prototype);
	//...
	return err;
}
CustomError4.prototype = Object.create(Error.prototype);

var e4 = new CustomError4();
log( e4 instanceof CustomError4 ) //true
log( e4 instanceof Error ) //true

An internal Error instance is another way but while it has the Error.prototype, it suffers the lack of a proper constructor (which will have name "Error") and the instance will not be an instance of the custom error object. However, I like the idea of setting the prototype of the internal Error instance to the CustomError prototype -- still it needs the proper constructor.

.setPrototypeOf() sets both the prototype and the constructor. So, for CustomError.prototype = Object.create(Error.prototype) you also need to set CustomError.prototype.constructor = CustomError;. This way, there will be quite no difference between implementations.

Actually I wrote a tester module for this. You can try it out in Node and various browsers. This also includes other implementations widely used. You can compare them by viewing the result output in console and generated HTML.

This test confirms that (with the constructor assignment), both .setPrototypeOf and Object.create implementations produce quite the same result. So it's really a preference to use one of them.

So combining them all; (with a little over-kill) this should be the best way to implement a custom error.

I'll update my SO answer to reflect these, soon. Thanks for this nice conversation. Pls go on if you have anything more to add/discuss.

--
Note: BTW, I don't care for supporting old IE versions so I'd still prefer .setPrototypeOf in other contexts too. 😎