token_get_all.php

<?php

$SRC = file_get_contents($argv[1]);

$TOKENS = token_get_all($SRC);

function safecrlf(& $str){
    $hascrlf = preg_match('/[\r\n]/',$str);
    $str = addcslashes($str,"\r\n\t ");
    return $hascrlf;
}

foreach($TOKENS as $token1 ){
    $crlf = false;
    if( is_string($token1) ){
        $crlf = safecrlf($token1);
        print "\"$token1\" ";
    } else {
        $mn = token_name($token1[0]);
        $crlf = safecrlf($token1[1]);
        print "<$mn>\"${token1[1]}\" ";
    }
    if($crlf){
        print "\n";
    }
}

txt

<T_OPEN_TAG>"<?php\n" 
<T_WHITESPACE>"\n" 
<T_VARIABLE>"$SRC" <T_WHITESPACE>"\ " "=" <T_WHITESPACE>"\ " <T_STRING>"file_get_contents" "(" <T_VARIABLE>"$argv" "[" <T_LNUMBER>"1" "]" ")" ";" <T_WHITESPACE>"\n\n" 
<T_VARIABLE>"$TOKENS" <T_WHITESPACE>"\ " "=" <T_WHITESPACE>"\ " <T_STRING>"token_get_all" "(" <T_VARIABLE>"$SRC" ")" ";" <T_WHITESPACE>"\n\n" 
<T_FUNCTION>"function" <T_WHITESPACE>"\ " <T_STRING>"safecrlf" "(" "&" <T_WHITESPACE>"\ " <T_VARIABLE>"$str" ")" "{" <T_WHITESPACE>"\n\ \ \ \ " 
<T_VARIABLE>"$hascrlf" <T_WHITESPACE>"\ " "=" <T_WHITESPACE>"\ " <T_STRING>"preg_match" "(" <T_CONSTANT_ENCAPSED_STRING>"'/[\r\n]/'" "," <T_VARIABLE>"$str" ")" ";" <T_WHITESPACE>"\n\ \ \ \ " 
<T_VARIABLE>"$str" <T_WHITESPACE>"\ " "=" <T_WHITESPACE>"\ " <T_STRING>"addcslashes" "(" <T_VARIABLE>"$str" "," <T_CONSTANT_ENCAPSED_STRING>""\r\n\t\ "" ")" ";" <T_WHITESPACE>"\n\ \ \ \ " 
<T_RETURN>"return" <T_WHITESPACE>"\ " <T_VARIABLE>"$hascrlf" ";" <T_WHITESPACE>"\n" 
"}" <T_WHITESPACE>"\n\n" 
<T_FOREACH>"foreach" "(" <T_VARIABLE>"$TOKENS" <T_WHITESPACE>"\ " <T_AS>"as" <T_WHITESPACE>"\ " <T_VARIABLE>"$token1" <T_WHITESPACE>"\ " ")" "{" <T_WHITESPACE>"\n\ \ \ \ " 
<T_VARIABLE>"$crlf" <T_WHITESPACE>"\ " "=" <T_WHITESPACE>"\ " <T_STRING>"false" ";" <T_WHITESPACE>"\n\ \ \ \ " 
<T_IF>"if" "(" <T_WHITESPACE>"\ " <T_STRING>"is_string" "(" <T_VARIABLE>"$token1" ")" <T_WHITESPACE>"\ " ")" "{" <T_WHITESPACE>"\n\ \ \ \ \ \ \ \ " 
<T_VARIABLE>"$crlf" <T_WHITESPACE>"\ " "=" <T_WHITESPACE>"\ " <T_STRING>"safecrlf" "(" <T_VARIABLE>"$token1" ")" ";" <T_WHITESPACE>"\n\ \ \ \ \ \ \ \ " 
<T_PRINT>"print" <T_WHITESPACE>"\ " """ <T_ENCAPSED_AND_WHITESPACE>"\"" <T_VARIABLE>"$token1" <T_ENCAPSED_AND_WHITESPACE>"\"\ " """ ";" <T_WHITESPACE>"\n\ \ \ \ " 
"}" <T_WHITESPACE>"\ " <T_ELSE>"else" <T_WHITESPACE>"\ " "{" <T_WHITESPACE>"\n\ \ \ \ \ \ \ \ " 
<T_VARIABLE>"$mn" <T_WHITESPACE>"\ " "=" <T_WHITESPACE>"\ " <T_STRING>"token_name" "(" <T_VARIABLE>"$token1" "[" <T_LNUMBER>"0" "]" ")" ";" <T_WHITESPACE>"\n\ \ \ \ \ \ \ \ " 
<T_VARIABLE>"$crlf" <T_WHITESPACE>"\ " "=" <T_WHITESPACE>"\ " <T_STRING>"safecrlf" "(" <T_VARIABLE>"$token1" "[" <T_LNUMBER>"1" "]" ")" ";" <T_WHITESPACE>"\n\ \ \ \ \ \ \ \ " 
<T_PRINT>"print" <T_WHITESPACE>"\ " """ <T_ENCAPSED_AND_WHITESPACE>"<" <T_VARIABLE>"$mn" <T_ENCAPSED_AND_WHITESPACE>">\"" <T_DOLLAR_OPEN_CURLY_BRACES>"${" <T_STRING_VARNAME>"token1" "[" <T_LNUMBER>"1" "]" "}" <T_ENCAPSED_AND_WHITESPACE>"\"\ " """ ";" <T_WHITESPACE>"\n\ \ \ \ " 
"}" <T_WHITESPACE>"\n\ \ \ \ " 
<T_IF>"if" "(" <T_VARIABLE>"$crlf" ")" "{" <T_WHITESPACE>"\n\ \ \ \ \ \ \ \ " 
<T_PRINT>"print" <T_WHITESPACE>"\ " <T_CONSTANT_ENCAPSED_STRING>""\n"" ";" <T_WHITESPACE>"\n\ \ \ \ " 
"}" <T_WHITESPACE>"\n" 
"}"