v2 ahora si verifica correctamente casos como ([{}()]) y más complejos, sin embargo hay que recorrer todo el string.

balanced_breackest_v2.js

var input1 = "{[()]}";
var input2 = "{[({}])}";
var input3 = "{{[[(())]]}}";
var input4 = "{[({{}[((({({})}))]}]])}";
var input5 = "([{}()])";
var input6 = "([{}()]()[])";

const matches = {
  "{": "}",
  "(": ")",
  "[": "]"
};

function isBalanced(input) {
  if (input.length % 2 !== 0) return "NO";
  openBraches = [];
  for (let c of input) {
    if (matches[c]) {
      openBraches.push(c);
    } else {
      if (matches[openBraches.pop()] != c) return "NO";
    }
  }

  return "YES";
}

function isBalancedR(input, openBraches = []) {
  if (input == "") return "YES";
  let c = input.substr(0, 1);
  if (matches[c]) {
    openBraches.push(c);
    return isBalancedR(input.substr(1), openBraches);
  } else {
    return matches[openBraches.pop()] != c ? "NO" : isBalancedR(input.substr(1), openBraches);
  }
}

console.log(isBalancedR(input2));
console.log(isBalancedR(input6));