Skip to content

Instantly share code, notes, and snippets.

@clytras

clytras/PHP Tokens.md

Last active December 3, 2021 00:12
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save clytras/169931c574c5061bdc1f3458fbffbd53 to your computer and use it in GitHub Desktop.
Save clytras/169931c574c5061bdc1f3458fbffbd53 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
_Extract php.net tokens.js
/**
* This is a snippet to extract all tokens from https://www.php.net/manual/en/tokens.php
* and generate a markdown table matching a json object with the values.
*
* To use it, visit https://www.php.net/manual/en/tokens.php, add this snippet to devtools snippets and run it.
*/
console.clear();
const tokens = {"T_THROW":258,"T_INCLUDE":260,"T_INCLUDE_ONCE":261,"T_REQUIRE":262,"T_REQUIRE_ONCE":263,"T_LOGICAL_OR":264,"T_LOGICAL_XOR":265,"T_LOGICAL_AND":266,"T_PRINT":267,"T_YIELD":268,"T_DOUBLE_ARROW":269,"T_YIELD_FROM":270,"T_PLUS_EQUAL":271,"T_MINUS_EQUAL":272,"T_MUL_EQUAL":273,"T_DIV_EQUAL":274,"T_CONCAT_EQUAL":275,"T_MOD_EQUAL":276,"T_AND_EQUAL":277,"T_OR_EQUAL":278,"T_XOR_EQUAL":279,"T_SL_EQUAL":280,"T_SR_EQUAL":281,"T_POW_EQUAL":282,"T_COALESCE_EQUAL":283,"T_COALESCE":284,"T_BOOLEAN_OR":285,"T_BOOLEAN_AND":286,"T_IS_EQUAL":287,"T_IS_NOT_EQUAL":288,"T_IS_IDENTICAL":289,"T_IS_NOT_IDENTICAL":290,"T_SPACESHIP":291,"T_IS_SMALLER_OR_EQUAL":292,"T_IS_GREATER_OR_EQUAL":293,"T_SL":294,"T_SR":295,"T_INSTANCEOF":296,"T_INT_CAST":297,"T_DOUBLE_CAST":298,"T_STRING_CAST":299,"T_ARRAY_CAST":300,"T_OBJECT_CAST":301,"T_BOOL_CAST":302,"T_UNSET_CAST":303,"T_POW":304,"T_CLONE":305,"T_ELSEIF":307,"T_ELSE":308,"T_LNUMBER":309,"T_DNUMBER":310,"T_STRING":311,"T_NAME_FULLY_QUALIFIED":312,"T_VARIABLE":315,"T_INLINE_HTML":316,"T_ENCAPSED_AND_WHITESPACE":317,"T_CONSTANT_ENCAPSED_STRING":318,"T_STRING_VARNAME":319,"T_NUM_STRING":320,"T_EVAL":321,"T_NEW":322,"T_EXIT":323,"T_IF":324,"T_ENDIF":325,"T_ECHO":326,"T_DO":327,"T_WHILE":328,"T_ENDWHILE":329,"T_FOR":330,"T_ENDFOR":331,"T_FOREACH":332,"T_ENDFOREACH":333,"T_DECLARE":334,"T_ENDDECLARE":335,"T_AS":336,"T_SWITCH":337,"T_ENDSWITCH":338,"T_CASE":339,"T_DEFAULT":340,"T_MATCH":341,"T_BREAK":342,"T_CONTINUE":343,"T_GOTO":344,"T_FUNCTION":345,"T_FN":346,"T_CONST":347,"T_RETURN":348,"T_TRY":349,"T_CATCH":350,"T_FINALLY":351,"T_USE":352,"T_INSTEADOF":353,"T_GLOBAL":354,"T_STATIC":355,"T_ABSTRACT":356,"T_FINAL":357,"T_PRIVATE":358,"T_PROTECTED":359,"T_PUBLIC":360,"T_VAR":361,"T_UNSET":362,"T_ISSET":363,"T_EMPTY":364,"T_HALT_COMPILER":365,"T_CLASS":366,"T_TRAIT":367,"T_INTERFACE":368,"T_EXTENDS":369,"T_IMPLEMENTS":370,"T_NAMESPACE":371,"T_LIST":372,"T_ARRAY":373,"T_CALLABLE":374,"T_LINE":375,"T_FILE":376,"T_DIR":377,"T_CLASS_C":378,"T_TRAIT_C":379,"T_METHOD_C":380,"T_FUNC_C":381,"T_NS_C":382,"T_ATTRIBUTE":383,"T_INC":384,"T_DEC":385,"T_OBJECT_OPERATOR":386,"T_NULLSAFE_OBJECT_OPERATOR":387,"T_COMMENT":388,"T_DOC_COMMENT":389,"T_OPEN_TAG":390,"T_OPEN_TAG_WITH_ECHO":391,"T_CLOSE_TAG":392,"T_WHITESPACE":393,"T_START_HEREDOC":394,"T_END_HEREDOC":395,"T_DOLLAR_OPEN_CURLY_BRACES":396,"T_CURLY_OPEN":397,"T_DOUBLE_COLON":398,"T_PAAMAYIM_NEKUDOTAYIM":398,"T_NS_SEPARATOR":399,"T_ELLIPSIS":400,"T_BAD_CHARACTER":401};
const all = {};
const rows = [...document.querySelectorAll('table tbody tr')];
let maxSyntaxLen = 0;
let maxRefLen = 0;
for (let row of rows) {
const tds = row.querySelectorAll('td');
const name = tds[0].textContent;
const syntax = tds[1].textContent.trim();
const refNodes = tds[2].querySelectorAll('a, code, strong');
const ref = [];
// Get Reference markdown text with links
for (let refNode of refNodes) {
let text = refNode.textContent.replaceAll(/\s+/g, ' ');
if (refNode instanceof HTMLAnchorElement) {
text = `[${text}](${refNode.href})`;
} else if (refNode.tagName === 'CODE') {
text = text.length && `\`${text}\``;
} else if (refNode.tagName === 'STRONG') {
text = text.length && `**${text}**`;
}
ref.push(text);
}
let reference = ref.join(' ');
maxRefLen = Math.max(maxRefLen, reference.length);
maxSyntaxLen = Math.max(maxSyntaxLen, syntax.length);
all[name] = { name, syntax, reference };
}
let md = '';
for (let id in tokens) {
if (id in all) {
const token = all[id];
const name = `**${id}**`.padEnd(30);
const value = tokens[id];
const dec = value.toString().padStart(6);
const hex = `0x${value.toString(16).toUpperCase()}`.padStart(6);
// const oct = `0${value.toString(8)}`.padEnd(6);
let syntax = token.syntax.length ? `\`${token.syntax}\`` : '';
syntax = `${syntax.padEnd(maxSyntaxLen + 2)}`;
const ref = token.reference;
md += `| ${name} | ${dec} | ${hex} | ${syntax} | ${ref} |\n`;
} else {
console.warn('token not found', id);
}
}
console.log(md);
Raw
PHP Tokens.md

PHP Token names with values

This is a list of all PHP 8.0 tokens and their represented values in decimal and hexadecimal. The list exists in the php.net site here https://www.php.net/manual/en/tokens.php but there are no values because these values may change for new releases. When writing token parsing code, we need token names and values, and I don't want to dig into the source code to get the values.

Beware that many, if not all values, may be different along version; PHP 7 may not have the same values with PHP 8. Always use the constants, this is just for reference mainly for debugging.

Blog link https://lytrax.io/blog/dev/php-tokens

Gist link https://gist.github.com/clytras/169931c574c5061bdc1f3458fbffbd53

So here is a list of all PHP 8.0 tokens and with their values.

Token Value Hex Syntax Reference
T_THROW 258 0x102 throw Exceptions
T_INCLUDE 260 0x104 include() include
T_INCLUDE_ONCE 261 0x105 include_once() include_once
T_REQUIRE 262 0x106 require() require
T_REQUIRE_ONCE 263 0x107 require_once() require_once
T_LOGICAL_OR 264 0x108 or logical operators
T_LOGICAL_XOR 265 0x109 xor logical operators
T_LOGICAL_AND 266 0x10A and logical operators
T_PRINT 267 0x10B print() print
T_YIELD 268 0x10C yield generators
T_DOUBLE_ARROW 269 0x10D => array syntax
T_YIELD_FROM 270 0x10E yield from generators
T_PLUS_EQUAL 271 0x10F += assignment operators
T_MINUS_EQUAL 272 0x110 -= assignment operators
T_MUL_EQUAL 273 0x111 *= assignment operators
T_DIV_EQUAL 274 0x112 /= assignment operators
T_CONCAT_EQUAL 275 0x113 .= assignment operators
T_MOD_EQUAL 276 0x114 %= assignment operators
T_AND_EQUAL 277 0x115 &= assignment operators
T_OR_EQUAL 278 0x116 |= assignment operators
T_XOR_EQUAL 279 0x117 ^= assignment operators
T_SL_EQUAL 280 0x118 <<= assignment operators
T_SR_EQUAL 281 0x119 >>= assignment operators
T_POW_EQUAL 282 0x11A **= assignment operators
T_COALESCE_EQUAL 283 0x11B ??= assignment operators
T_COALESCE 284 0x11C ?? comparison operators
T_BOOLEAN_OR 285 0x11D || logical operators
T_BOOLEAN_AND 286 0x11E && logical operators
T_IS_EQUAL 287 0x11F == comparison operators
T_IS_NOT_EQUAL 288 0x120 != or <> comparison operators
T_IS_IDENTICAL 289 0x121 === comparison operators
T_IS_NOT_IDENTICAL 290 0x122 !== comparison operators
T_SPACESHIP 291 0x123 <=> comparison operators
T_IS_SMALLER_OR_EQUAL 292 0x124 <= comparison operators
T_IS_GREATER_OR_EQUAL 293 0x125 >= comparison operators
T_SL 294 0x126 << bitwise operators
T_SR 295 0x127 >> bitwise operators
T_INSTANCEOF 296 0x128 instanceof type operators
T_INT_CAST 297 0x129 (int) or (integer) type-casting
T_DOUBLE_CAST 298 0x12A (real), (double) or (float) type-casting
T_STRING_CAST 299 0x12B (string) type-casting
T_ARRAY_CAST 300 0x12C (array) type-casting
T_OBJECT_CAST 301 0x12D (object) type-casting
T_BOOL_CAST 302 0x12E (bool) or (boolean) type-casting
T_UNSET_CAST 303 0x12F (unset) type-casting
T_POW 304 0x130 ** arithmetic operators
T_CLONE 305 0x131 clone classes and objects
T_ELSEIF 307 0x133 elseif elseif
T_ELSE 308 0x134 else else
T_LNUMBER 309 0x135 123, 012, 0x1ac, etc. integers
T_DNUMBER 310 0x136 0.12, etc. floating point numbers
T_STRING 311 0x137 parent, self, etc. parent self T_CONSTANT_ENCAPSED_STRING T_CONSTANT_ENCAPSED_STRING
T_NAME_FULLY_QUALIFIED 312 0x138 \App\Namespace namespaces
T_VARIABLE 315 0x13B $foo variables
T_INLINE_HTML 316 0x13C text outside PHP
T_ENCAPSED_AND_WHITESPACE 317 0x13D " $a" constant part of string with variables
T_CONSTANT_ENCAPSED_STRING 318 0x13E "foo" or 'bar' string syntax
T_STRING_VARNAME 319 0x13F "${a complex variable parsed syntax
T_NUM_STRING 320 0x140 "$a[0]" numeric array index inside string
T_EVAL 321 0x141 eval() eval()
T_NEW 322 0x142 new classes and objects
T_EXIT 323 0x143 exit or die exit() die()
T_IF 324 0x144 if if
T_ENDIF 325 0x145 endif if alternative syntax
T_ECHO 326 0x146 echo echo
T_DO 327 0x147 do do..while
T_WHILE 328 0x148 while while do..while
T_ENDWHILE 329 0x149 endwhile while alternative syntax
T_FOR 330 0x14A for for
T_ENDFOR 331 0x14B endfor for alternative syntax
T_FOREACH 332 0x14C foreach foreach
T_ENDFOREACH 333 0x14D endforeach foreach alternative syntax
T_DECLARE 334 0x14E declare declare
T_ENDDECLARE 335 0x14F enddeclare declare alternative syntax
T_AS 336 0x150 as foreach
T_SWITCH 337 0x151 switch switch
T_ENDSWITCH 338 0x152 endswitch switch alternative syntax
T_CASE 339 0x153 case switch
T_DEFAULT 340 0x154 default switch
T_MATCH 341 0x155 match match
T_BREAK 342 0x156 break break
T_CONTINUE 343 0x157 continue continue
T_GOTO 344 0x158 goto goto
T_FUNCTION 345 0x159 function functions
T_FN 346 0x15A fn arrow functions
T_CONST 347 0x15B const class constants
T_RETURN 348 0x15C return returning values
T_TRY 349 0x15D try Exceptions
T_CATCH 350 0x15E catch Exceptions
T_FINALLY 351 0x15F finally Exceptions
T_USE 352 0x160 use namespaces
T_INSTEADOF 353 0x161 insteadof Traits
T_GLOBAL 354 0x162 global variable scope
T_STATIC 355 0x163 static variable scope
T_ABSTRACT 356 0x164 abstract Class Abstraction
T_FINAL 357 0x165 final Final Keyword
T_PRIVATE 358 0x166 private classes and objects
T_PROTECTED 359 0x167 protected classes and objects
T_PUBLIC 360 0x168 public classes and objects
T_VAR 361 0x169 var classes and objects
T_UNSET 362 0x16A unset() unset()
T_ISSET 363 0x16B isset() isset()
T_EMPTY 364 0x16C empty empty()
T_HALT_COMPILER 365 0x16D __halt_compiler() __halt_compiler
T_CLASS 366 0x16E class classes and objects
T_TRAIT 367 0x16F trait Traits
T_INTERFACE 368 0x170 interface Object Interfaces
T_EXTENDS 369 0x171 extends extends classes and objects
T_IMPLEMENTS 370 0x172 implements Object Interfaces
T_NAMESPACE 371 0x173 namespace namespaces
T_LIST 372 0x174 list() list()
T_ARRAY 373 0x175 array() array() array syntax
T_CALLABLE 374 0x176 callable callable
T_LINE 375 0x177 __LINE__ magic constants
T_FILE 376 0x178 __FILE__ magic constants
T_DIR 377 0x179 __DIR__ magic constants
T_CLASS_C 378 0x17A __CLASS__ magic constants
T_TRAIT_C 379 0x17B __TRAIT__ TRAIT
T_METHOD_C 380 0x17C __METHOD__ magic constants
T_FUNC_C 381 0x17D __FUNCTION__ magic constants
T_NS_C 382 0x17E __NAMESPACE__ namespaces
T_ATTRIBUTE 383 0x17F #[ attributes
T_INC 384 0x180 ++ incrementing/decrementing operators
T_DEC 385 0x181 -- incrementing/decrementing operators
T_OBJECT_OPERATOR 386 0x182 -> classes and objects
T_NULLSAFE_OBJECT_OPERATOR 387 0x183 ?-> classes and objects
T_COMMENT 388 0x184 // or #, and /* */ comments
T_DOC_COMMENT 389 0x185 /** */ PHPDoc style comments
T_OPEN_TAG 390 0x186 <?php, <? or <% escaping from HTML
T_OPEN_TAG_WITH_ECHO 391 0x187 <?= or <%= escaping from HTML
T_CLOSE_TAG 392 0x188 ?> or %> escaping from HTML
T_WHITESPACE 393 0x189 \t \r\n
T_START_HEREDOC 394 0x18A <<< heredoc syntax
T_END_HEREDOC 395 0x18B heredoc syntax
T_DOLLAR_OPEN_CURLY_BRACES 396 0x18C ${ complex variable parsed syntax
T_CURLY_OPEN 397 0x18D {$ complex variable parsed syntax
T_DOUBLE_COLON 398 0x18E :: T_PAAMAYIM_NEKUDOTAYIM T_PAAMAYIM_NEKUDOTAYIM
T_PAAMAYIM_NEKUDOTAYIM 398 0x18E :: :: T_DOUBLE_COLON T_DOUBLE_COLON
T_NS_SEPARATOR 399 0x18F \ namespaces
T_ELLIPSIS 400 0x190 ... function arguments
T_BAD_CHARACTER 401 0x191
Raw
PHP Tokens.txt
PHP Token names with values in plain text.
Blog link https://lytrax.io/blog/dev/php-tokens
Gist link https://gist.github.com/clytras/169931c574c5061bdc1f3458fbffbd53
Token DEC HEX OCT
------------------------------------------------
T_THROW 258 0x102 0402
T_INCLUDE 260 0x104 0404
T_INCLUDE_ONCE 261 0x105 0405
T_REQUIRE 262 0x106 0406
T_REQUIRE_ONCE 263 0x107 0407
T_LOGICAL_OR 264 0x108 0410
T_LOGICAL_XOR 265 0x109 0411
T_LOGICAL_AND 266 0x10A 0412
T_PRINT 267 0x10B 0413
T_YIELD 268 0x10C 0414
T_DOUBLE_ARROW 269 0x10D 0415
T_YIELD_FROM 270 0x10E 0416
T_PLUS_EQUAL 271 0x10F 0417
T_MINUS_EQUAL 272 0x110 0420
T_MUL_EQUAL 273 0x111 0421
T_DIV_EQUAL 274 0x112 0422
T_CONCAT_EQUAL 275 0x113 0423
T_MOD_EQUAL 276 0x114 0424
T_AND_EQUAL 277 0x115 0425
T_OR_EQUAL 278 0x116 0426
T_XOR_EQUAL 279 0x117 0427
T_SL_EQUAL 280 0x118 0430
T_SR_EQUAL 281 0x119 0431
T_POW_EQUAL 282 0x11A 0432
T_COALESCE_EQUAL 283 0x11B 0433
T_COALESCE 284 0x11C 0434
T_BOOLEAN_OR 285 0x11D 0435
T_BOOLEAN_AND 286 0x11E 0436
T_IS_EQUAL 287 0x11F 0437
T_IS_NOT_EQUAL 288 0x120 0440
T_IS_IDENTICAL 289 0x121 0441
T_IS_NOT_IDENTICAL 290 0x122 0442
T_SPACESHIP 291 0x123 0443
T_IS_SMALLER_OR_EQUAL 292 0x124 0444
T_IS_GREATER_OR_EQUAL 293 0x125 0445
T_SL 294 0x126 0446
T_SR 295 0x127 0447
T_INSTANCEOF 296 0x128 0450
T_INT_CAST 297 0x129 0451
T_DOUBLE_CAST 298 0x12A 0452
T_STRING_CAST 299 0x12B 0453
T_ARRAY_CAST 300 0x12C 0454
T_OBJECT_CAST 301 0x12D 0455
T_BOOL_CAST 302 0x12E 0456
T_UNSET_CAST 303 0x12F 0457
T_POW 304 0x130 0460
T_CLONE 305 0x131 0461
T_ELSEIF 307 0x133 0463
T_ELSE 308 0x134 0464
T_LNUMBER 309 0x135 0465
T_DNUMBER 310 0x136 0466
T_STRING 311 0x137 0467
T_NAME_FULLY_QUALIFIED 312 0x138 0470
T_VARIABLE 315 0x13B 0473
T_INLINE_HTML 316 0x13C 0474
T_ENCAPSED_AND_WHITESPACE 317 0x13D 0475
T_CONSTANT_ENCAPSED_STRING 318 0x13E 0476
T_STRING_VARNAME 319 0x13F 0477
T_NUM_STRING 320 0x140 0500
T_EVAL 321 0x141 0501
T_NEW 322 0x142 0502
T_EXIT 323 0x143 0503
T_IF 324 0x144 0504
T_ENDIF 325 0x145 0505
T_ECHO 326 0x146 0506
T_DO 327 0x147 0507
T_WHILE 328 0x148 0510
T_ENDWHILE 329 0x149 0511
T_FOR 330 0x14A 0512
T_ENDFOR 331 0x14B 0513
T_FOREACH 332 0x14C 0514
T_ENDFOREACH 333 0x14D 0515
T_DECLARE 334 0x14E 0516
T_ENDDECLARE 335 0x14F 0517
T_AS 336 0x150 0520
T_SWITCH 337 0x151 0521
T_ENDSWITCH 338 0x152 0522
T_CASE 339 0x153 0523
T_DEFAULT 340 0x154 0524
T_MATCH 341 0x155 0525
T_BREAK 342 0x156 0526
T_CONTINUE 343 0x157 0527
T_GOTO 344 0x158 0530
T_FUNCTION 345 0x159 0531
T_FN 346 0x15A 0532
T_CONST 347 0x15B 0533
T_RETURN 348 0x15C 0534
T_TRY 349 0x15D 0535
T_CATCH 350 0x15E 0536
T_FINALLY 351 0x15F 0537
T_USE 352 0x160 0540
T_INSTEADOF 353 0x161 0541
T_GLOBAL 354 0x162 0542
T_STATIC 355 0x163 0543
T_ABSTRACT 356 0x164 0544
T_FINAL 357 0x165 0545
T_PRIVATE 358 0x166 0546
T_PROTECTED 359 0x167 0547
T_PUBLIC 360 0x168 0550
T_VAR 361 0x169 0551
T_UNSET 362 0x16A 0552
T_ISSET 363 0x16B 0553
T_EMPTY 364 0x16C 0554
T_HALT_COMPILER 365 0x16D 0555
T_CLASS 366 0x16E 0556
T_TRAIT 367 0x16F 0557
T_INTERFACE 368 0x170 0560
T_EXTENDS 369 0x171 0561
T_IMPLEMENTS 370 0x172 0562
T_NAMESPACE 371 0x173 0563
T_LIST 372 0x174 0564
T_ARRAY 373 0x175 0565
T_CALLABLE 374 0x176 0566
T_LINE 375 0x177 0567
T_FILE 376 0x178 0570
T_DIR 377 0x179 0571
T_CLASS_C 378 0x17A 0572
T_TRAIT_C 379 0x17B 0573
T_METHOD_C 380 0x17C 0574
T_FUNC_C 381 0x17D 0575
T_NS_C 382 0x17E 0576
T_ATTRIBUTE 383 0x17F 0577
T_INC 384 0x180 0600
T_DEC 385 0x181 0601
T_OBJECT_OPERATOR 386 0x182 0602
T_NULLSAFE_OBJECT_OPERATOR 387 0x183 0603
T_COMMENT 388 0x184 0604
T_DOC_COMMENT 389 0x185 0605
T_OPEN_TAG 390 0x186 0606
T_OPEN_TAG_WITH_ECHO 391 0x187 0607
T_CLOSE_TAG 392 0x188 0610
T_WHITESPACE 393 0x189 0611
T_START_HEREDOC 394 0x18A 0612
T_END_HEREDOC 395 0x18B 0613
T_DOLLAR_OPEN_CURLY_BRACES 396 0x18C 0614
T_CURLY_OPEN 397 0x18D 0615
T_DOUBLE_COLON 398 0x18E 0616
T_PAAMAYIM_NEKUDOTAYIM 398 0x18E 0616
T_NS_SEPARATOR 399 0x18F 0617
T_ELLIPSIS 400 0x190 0620
T_BAD_CHARACTER 401 0x191 0621
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment