The algorithm

test.php

<?php

function check($str) {
    $length = strlen($str);
    $rules = array('NULL', '()', '[]', '[NULL]','(NULL)', '()[]');
    $i = 0;
    while ($i <= $length/2) {
        foreach ($rules as  $rule) {
            $str = str_replace($rule, '', $str);
        }

        if (empty($str)) {
            return true;
        }

        $i++;
    }

    return false;
}

assert(check('') === TRUE, '[()] => TRUE');
assert(check('[()]') === TRUE, '[()] => TRUE');
assert(check('(()[])') === TRUE,'(()[]) => TRUE');
assert(check('([)]') === FALSE, '([)] => FALSE');
assert(check('((') === FALSE, '(( => FALSE');
assert(check('[(()])') === FALSE, '[(()]) => FALSE');


assert(check("([(([[(([]))]]))])") === TRUE, "([(([[(([]))]]))]) => TRUE");
assert(check("[](()()[[]])()[]([])") === TRUE, "[](()()[[]])()[]([]) => TRUE");
assert(check("([((([(([]))])))))])") === FALSE, "([((([(([]))])))))]) => FALSE");
assert(check("[](()()[[]])[][[([])") === FALSE, "[](()()[[]])[][[([]) => FALSE");