Recursively sort all layers in Illustrator alphabetically

sortLayers.jsx

var deep = true, // Set to false if you don't want to recursively sort layers within layers too
    isLowerFirst = true; // Set it to false if you want to place the layers with the first uppercase letter above

// Convert ILST collection into standard Array so we can use Array methods
function get(type, parent) {
  if (arguments.length == 1 || !parent) parent = app.activeDocument;
  var result = [];
  if (!parent[type]) return result;
  for (var i = 0; i < parent[type].length; i++) result.push(parent[type][i]);
  return result;
}

// Polyfill for Array.forEach
Array.prototype.forEach = function (callback) {
  for (var i = 0; i < this.length; i++) callback(this[i], i, this);
};

function sortLayersInside(parent) {
  get("layers", parent)
    .sort(function (a, b) {
      var aZero = a.name.charAt(0),
          bZero = b.name.charAt(0);
      
      if (a.name === b.name) return 0;

      if (aZero.toLowerCase() === bZero.toLowerCase()) {
        if (aZero !== aZero.toUpperCase() && bZero === bZero.toUpperCase()) {
          return isLowerFirst ? -1 : 1;
        };
        if (aZero === aZero.toUpperCase() && bZero !== bZero.toUpperCase()){
          return isLowerFirst ? 1 : -1;
        };
      }

      return a.name.toLowerCase().localeCompare(b.name.toLowerCase());
    })
    .forEach(function (layer) {
      layer.zOrder(ZOrderMethod.SENDTOBACK);
      if (layer.layers && layer.layers.length && deep) sortLayersInside(layer);
    });
}

sortLayersInside();

Nice! I thought I could clean it up and make it shorter or make the logic less cluttered in the sort function itself, but I think I might be overcomplicating things at this point. I tweaked your revisions to this:

var deep = true, // Set to false if you don't want to recursively sort layers within layers too
    isLowerFirst = true; // Set it to false if you want to place the layers with the first uppercase letter above

// Convert ILST collection into standard Array so we can use Array methods
function get(type, parent) {
    if (arguments.length == 1 || !parent) parent = app.activeDocument;
    var result = [];
    if (!parent[type]) return result;
    for (var i = 0; i < parent[type].length; i++) result.push(parent[type][i]);
    return result;
}

// Polyfill for Array.forEach
Array.prototype.forEach = function (callback) {
    for (var i = 0; i < this.length; i++) callback(this[i], i, this);
};

function sortLayersInside(parent) {
    function isUpperCase(text) {
        return /[A-Z]/.test(text.charAt(0))
    }
    function isLowerCase(text) {
        return /[a-z]/.test(text.charAt(0))
    }
    function hasMixedCase(a, b) {
        if (a.charAt(0).toLowerCase() !== b.charAt(0).toLowerCase()) return false;
        return isUpperCase(a) && isLowerCase(b) ? true : isLowerCase(a) && isUpperCase(b);
    }
    function handleMixedCase(a, b) {
        var result = a === b ? 0 : isUpperCase(a) ? 1 : -1;
        return isLowerFirst ? result : result * -1;
    }
    get("layers", parent)
        .sort(function (a, b) {
            return hasMixedCase(a.name, b.name) ?
                handleMixedCase(a.name, b.name) :
                a.name.toLowerCase().localeCompare(b.name.toLowerCase())
        })
        .forEach(function (layer) {
            layer.zOrder(ZOrderMethod.SENDTOBACK);
            if (layer.layers && layer.layers.length && deep) sortLayersInside(layer);
        });
}
sortLayersInside();