myTry

"""
(venv) ionianironist@debian:~/PycharmProjects/untitled$ python lab.py
VAR_TOKEN a32
VAR_TOKEN b
VAR_TOKEN _67
VAR_TOKEN a_9
VAR_TOKEN DeF
VAR_TOKEN ifelse
VAR_TOKEN then


"""
import plex

# define patterns
letter = plex.Range("AZaz")
digit = plex.Range("09")

# the scanner lexicon - constructor argument is a list of (pattern,action ) tuples
lexicon = plex.Lexicon([
    ((letter | plex.Str("_")) + plex.Rep((letter | digit) | plex.Str("_")), "VAR_TOKEN"),
    (plex.Rep1(plex.Any(" \t\n")), plex.IGNORE)
    ])

with open("identifiers.txr", "r") as fp:
    scanner = plex.Scanner(lexicon, fp)

    while True:
        token, matched = scanner.read()

        if not token:
            break  # reached end-of-text (EOT)

        print(token, matched)