Quick Mask Mode

quickMask.jsx

activeDocument.quickMaskMode = true;
activeDocument.quickMaskMode = false;

//or

quickMask("set"); // or "clearEvent"

function quickMask(qmOpt) {
	var s2t = function (s) {
		return app.stringIDToTypeID(s);
	};
	var descriptor = new ActionDescriptor();
	var reference = new ActionReference();
	reference.putProperty( s2t( "property" ), s2t( "quickMask" ));
	reference.putEnumerated( s2t( "document" ), s2t( "ordinal" ), s2t( "targetEnum" ));
    descriptor.putReference(s2t("null"), reference);
    // "set" to enter | "clearEvent" to exit
	executeAction( s2t( qmOpt ), descriptor, DialogModes.NO );
}

//or

// Toggle QuickMask mode
app.activeDocument.quickMaskMode ^= 1