Additional token_get_all param

diff.patch

diff --git a/ext/tokenizer/tests/token_get_all_error.phpt b/ext/tokenizer/tests/token_get_all_error.phpt
index 29e97c3..5b455e9 100644
--- a/ext/tokenizer/tests/token_get_all_error.phpt
+++ b/ext/tokenizer/tests/token_get_all_error.phpt
@@ -19,7 +19,7 @@ var_dump( token_get_all());
 echo "-- Testing token_get_all() function with more than expected no. of arguments --\n";
 $source = '<?php ?>';
 $extra_arg = 10;
-var_dump( token_get_all($source, $extra_arg));
+var_dump( token_get_all($source, false, $extra_arg));
 
 echo "Done"
 ?>
@@ -28,10 +28,10 @@ echo "Done"
 
 -- Testing token_get_all() function with zero arguments --
 
-Warning: token_get_all() expects exactly 1 parameter, 0 given in %s on line %d
+Warning: token_get_all() expects at least 1 parameter, 0 given in %s on line %d
 NULL
 -- Testing token_get_all() function with more than expected no. of arguments --
 
-Warning: token_get_all() expects exactly 1 parameter, 2 given in %s on line %d
+Warning: token_get_all() expects at most 2 parameters, 3 given in %s on line %d
 NULL
 Done
diff --git a/ext/tokenizer/tests/token_get_all_variation20.phpt b/ext/tokenizer/tests/token_get_all_variation20.phpt
new file mode 100644
index 0000000..6b6e351
--- /dev/null
+++ b/ext/tokenizer/tests/token_get_all_variation20.phpt
@@ -0,0 +1,163 @@
+--TEST--
+Returning all tokens as arrays (second argument)
+--SKIPIF--
+<?php if (!extension_loaded("tokenizer")) print "skip"; ?>
+--FILE--
+<?php
+$code = <<<EOC
+<?php
+test   . some  $
+single ~ chars !
+EOC;
+
+var_dump(token_get_all($code, true));
+?>
+--EXPECT--
+array(16) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(372)
+    [1]=>
+    string(6) "<?php
+"
+    [2]=>
+    int(1)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(307)
+    [1]=>
+    string(4) "test"
+    [2]=>
+    int(2)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(375)
+    [1]=>
+    string(3) "   "
+    [2]=>
+    int(2)
+  }
+  [3]=>
+  array(3) {
+    [0]=>
+    int(46)
+    [1]=>
+    string(1) "."
+    [2]=>
+    int(2)
+  }
+  [4]=>
+  array(3) {
+    [0]=>
+    int(375)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(2)
+  }
+  [5]=>
+  array(3) {
+    [0]=>
+    int(307)
+    [1]=>
+    string(4) "some"
+    [2]=>
+    int(2)
+  }
+  [6]=>
+  array(3) {
+    [0]=>
+    int(375)
+    [1]=>
+    string(2) "  "
+    [2]=>
+    int(2)
+  }
+  [7]=>
+  array(3) {
+    [0]=>
+    int(36)
+    [1]=>
+    string(1) "$"
+    [2]=>
+    int(2)
+  }
+  [8]=>
+  array(3) {
+    [0]=>
+    int(375)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(2)
+  }
+  [9]=>
+  array(3) {
+    [0]=>
+    int(307)
+    [1]=>
+    string(6) "single"
+    [2]=>
+    int(3)
+  }
+  [10]=>
+  array(3) {
+    [0]=>
+    int(375)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(3)
+  }
+  [11]=>
+  array(3) {
+    [0]=>
+    int(126)
+    [1]=>
+    string(1) "~"
+    [2]=>
+    int(3)
+  }
+  [12]=>
+  array(3) {
+    [0]=>
+    int(375)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(3)
+  }
+  [13]=>
+  array(3) {
+    [0]=>
+    int(307)
+    [1]=>
+    string(5) "chars"
+    [2]=>
+    int(3)
+  }
+  [14]=>
+  array(3) {
+    [0]=>
+    int(375)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(3)
+  }
+  [15]=>
+  array(3) {
+    [0]=>
+    int(33)
+    [1]=>
+    string(1) "!"
+    [2]=>
+    int(3)
+  }
+}
diff --git a/ext/tokenizer/tokenizer.c b/ext/tokenizer/tokenizer.c
index afdd85b..4b7f3e5 100644
--- a/ext/tokenizer/tokenizer.c
+++ b/ext/tokenizer/tokenizer.c
@@ -40,6 +40,7 @@
 /* {{{ arginfo */
 ZEND_BEGIN_ARG_INFO_EX(arginfo_token_get_all, 0, 0, 1)
 	ZEND_ARG_INFO(0, source)
+	ZEND_ARG_INFO(0, all_tokens_as_arrays)
 ZEND_END_ARG_INFO()
 
 ZEND_BEGIN_ARG_INFO_EX(arginfo_token_name, 0, 0, 1)
@@ -101,7 +102,7 @@ PHP_MINFO_FUNCTION(tokenizer)
 }
 /* }}} */
 
-static void tokenize(zval *return_value TSRMLS_DC)
+static void tokenize(zval *return_value, zend_bool all_tokens_as_arrays TSRMLS_DC)
 {
 	zval token;
 	zval *keyword;
@@ -129,7 +130,7 @@ static void tokenize(zval *return_value TSRMLS_DC)
 				break;
 		}
 
-		if (token_type >= 256) {
+		if (all_tokens_as_arrays || token_type >= 256) {
 			MAKE_STD_ZVAL(keyword);
 			array_init(keyword);
 			add_next_index_long(keyword, token_type);
@@ -185,10 +186,11 @@ PHP_FUNCTION(token_get_all)
 	char *source = NULL;
 	int argc = ZEND_NUM_ARGS();
 	int source_len;
+	zend_bool all_tokens_as_arrays = 0;
 	zval source_z;
 	zend_lex_state original_lex_state;
 
-	if (zend_parse_parameters(argc TSRMLS_CC, "s", &source, &source_len) == FAILURE) {
+	if (zend_parse_parameters(argc TSRMLS_CC, "s|b", &source, &source_len, &all_tokens_as_arrays) == FAILURE) {
 		return;
 	}
 
@@ -202,7 +204,7 @@ PHP_FUNCTION(token_get_all)
 
 	LANG_SCNG(yy_state) = yycINITIAL;
 
-	tokenize(return_value TSRMLS_CC);
+	tokenize(return_value, all_tokens_as_arrays TSRMLS_CC);
 	
 	zend_restore_lexical_state(&original_lex_state TSRMLS_CC);
 	zval_dtor(&source_z);