pseudo lexical structure

lexical-structure.txt

## tokens
keyword
operator
identifier
literal
symbol

### keyword
"_"
"as"
"break"
"continue"
"else"
"enum"
"extern"
"fn"
"for"
"if"
"impl"
"in"
"let"
"loop"
"match"
"move"
"ref"
"return"
"self"
"Self"
"static"
"struct"
"super"
"trait"
"type"
"where"
"while"
"input"
"output"
"log"

### operator
"+"
"-"
"*"
"/"
"//"
"%"
"&"
"|"
"^"
"<<"
">>"
"!"
"&&"
"||"
"=="
"!="
"<"
">"
"<="
">="
"=" and every assignment operators
"?:"


### identifier
r"[A-Za-z_][A-Za-z_0-9]*"
r"[2-5]dArray"


### literal
bool_literal = "true" | "false"
dec_int_literal = r"[0-9]([0-9_]*[0-9])?"
bin_int_literal = r"0[bB][0-1]([0-1_]*[0-1])?"
oct_int_literal = r"0[oO][0-7]([0-7_]*[0-7])?"
hex_int_literal = r"0[xX][0-9a-fA-F]([0-9a-fA-F_]*[0-9a-fA-F])?"
float_literal = r"[0-9]([0-9_]*[0-9])?(.[0-9]([0-9_]*[0-9])?)*([eE][+-]?[0-9]([0-9_]*[0-9])?)?"
char_literal = r"'[^\\\'\n\r]|`escape`'"
str_literal = r"\"([^\\\"\n\r]|`escape`)*\""

// TODO : hex_float_literal

#### escape
\0
\b
\t
\n
\f
\r
\"
\'
\\
r"\u[0-9a-fA-F]{4}"


### symbol
"("
")"
"{"
"}"
"["
"]"
"<"
">"
"#"
"."
","
":"
"::"
"?"
";"
"->"