Tokenizes a PHP script with token_get_all(), and outputs a readable representation.

tokenize-php-script.php

<?php

/**
 * Tokenizes a PHP script with token_get_all(), and outputs a readable representation.
 *
 * Usage: tokenize-php-script.php FILE.php
 *
 * Sample output:
 *
 *    1 T_OPEN_TAG                 <?php\n
 *    2 T_WHITESPACE               \n
 *    3 T_DECLARE                  declare
 *    3                            (
 *    3 T_STRING                   strict_types
 *    3                            =
 *    3 T_LNUMBER                  1
 *    3                            )
 *    3                            ;
 *    3 T_WHITESPACE               \n\n
 *    5 T_NAMESPACE                namespace
 *    5 T_WHITESPACE                
 *    5 T_NAME_QUALIFIED           Brick\Math
 *    5                            ;
 *
 *    ...
 */

declare(strict_types=1);

if ($argc !== 2) {
    echo "Usage: {$argv[0]} FILE\n";
    exit(1);
}

$file = $argv[1];
$code = file_get_contents($file);

$line = 1;

$tokens = [];
$tokenNameLen = 0;

foreach (token_get_all($code) as $token) {
    if (is_array($token)) {
        [$type, $string, $line] = $token;

        $name = token_name($type);
        $tokens[] = [$name, $string, $line];

        $tokenNameLen = max($tokenNameLen, strlen($name));
    } else {
        $tokens[] = ['', $token, $line];
    }
}

$lineNumLen = strlen((string) $line);

foreach ($tokens as $token) {
    [$name, $string, $line] = $token;

    printf("%{$lineNumLen}s %-{$tokenNameLen}s %s\n", $line, $name, exportString($string));
}

function exportString(string $string): string
{
    $string = str_replace(["\r", "\n", "\t"], ['\r', '\n', '\t'], $string);

    return $string;
}