Skip to content

Instantly share code, notes, and snippets.

@NaPs

NaPs/parser.out Secret

Created March 4, 2012 22:10
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save NaPs/ed4b5152a707b0ad2696 to your computer and use it in GitHub Desktop.
Save NaPs/ed4b5152a707b0ad2696 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
parser.out
Created by PLY version 3.4 (http://www.dabeaz.com/ply)
Grammar
Rule 0 S' -> top
Rule 1 top -> section_content
Rule 2 assignation -> NAME ASSIGN value
Rule 3 assignation -> NAME ASSIGN list
Rule 4 value -> TEXT
Rule 5 value -> YES
Rule 6 value -> NO
Rule 7 value -> NUMBER
Rule 8 list -> value LIST_SEP EOL list_next
Rule 9 list -> value LIST_SEP list_next
Rule 10 list -> value LIST_SEP
Rule 11 list_next -> value LIST_SEP EOL list_next
Rule 12 list_next -> value LIST_SEP list_next
Rule 13 list_next -> value LIST_SEP
Rule 14 list_next -> value
Rule 15 section_content -> section_content EOL
Rule 16 section_content -> empty
Rule 17 section_content -> section_content assignation
Rule 18 section_content -> section_content section
Rule 19 section -> NAME LBRACE section_content RBRACE
Rule 20 empty -> <empty>
Terminals, with rules where they appear
ASSIGN : 2 3
EOL : 8 11 15
LBRACE : 19
LIST_SEP : 8 9 10 11 12 13
NAME : 2 3 19
NO : 6
NUMBER : 7
RBRACE : 19
TEXT : 4
YES : 5
error :
Nonterminals, with rules where they appear
assignation : 17
empty : 16
list : 3
list_next : 8 9 11 12
section : 18
section_content : 1 15 17 18 19
top : 0
value : 2 8 9 10 11 12 13 14
Parsing method: LALR
state 0
(0) S' -> . top
(1) top -> . section_content
(15) section_content -> . section_content EOL
(16) section_content -> . empty
(17) section_content -> . section_content assignation
(18) section_content -> . section_content section
(20) empty -> .
EOL reduce using rule 20 (empty -> .)
NAME reduce using rule 20 (empty -> .)
$end reduce using rule 20 (empty -> .)
section_content shift and go to state 1
top shift and go to state 2
empty shift and go to state 3
state 1
(1) top -> section_content .
(15) section_content -> section_content . EOL
(17) section_content -> section_content . assignation
(18) section_content -> section_content . section
(2) assignation -> . NAME ASSIGN value
(3) assignation -> . NAME ASSIGN list
(19) section -> . NAME LBRACE section_content RBRACE
$end reduce using rule 1 (top -> section_content .)
EOL shift and go to state 6
NAME shift and go to state 4
section shift and go to state 5
assignation shift and go to state 7
state 2
(0) S' -> top .
state 3
(16) section_content -> empty .
EOL reduce using rule 16 (section_content -> empty .)
NAME reduce using rule 16 (section_content -> empty .)
$end reduce using rule 16 (section_content -> empty .)
RBRACE reduce using rule 16 (section_content -> empty .)
state 4
(2) assignation -> NAME . ASSIGN value
(3) assignation -> NAME . ASSIGN list
(19) section -> NAME . LBRACE section_content RBRACE
ASSIGN shift and go to state 9
LBRACE shift and go to state 8
state 5
(18) section_content -> section_content section .
EOL reduce using rule 18 (section_content -> section_content section .)
NAME reduce using rule 18 (section_content -> section_content section .)
$end reduce using rule 18 (section_content -> section_content section .)
RBRACE reduce using rule 18 (section_content -> section_content section .)
state 6
(15) section_content -> section_content EOL .
EOL reduce using rule 15 (section_content -> section_content EOL .)
NAME reduce using rule 15 (section_content -> section_content EOL .)
$end reduce using rule 15 (section_content -> section_content EOL .)
RBRACE reduce using rule 15 (section_content -> section_content EOL .)
state 7
(17) section_content -> section_content assignation .
EOL reduce using rule 17 (section_content -> section_content assignation .)
NAME reduce using rule 17 (section_content -> section_content assignation .)
$end reduce using rule 17 (section_content -> section_content assignation .)
RBRACE reduce using rule 17 (section_content -> section_content assignation .)
state 8
(19) section -> NAME LBRACE . section_content RBRACE
(15) section_content -> . section_content EOL
(16) section_content -> . empty
(17) section_content -> . section_content assignation
(18) section_content -> . section_content section
(20) empty -> .
RBRACE reduce using rule 20 (empty -> .)
EOL reduce using rule 20 (empty -> .)
NAME reduce using rule 20 (empty -> .)
section_content shift and go to state 10
empty shift and go to state 3
state 9
(2) assignation -> NAME ASSIGN . value
(3) assignation -> NAME ASSIGN . list
(4) value -> . TEXT
(5) value -> . YES
(6) value -> . NO
(7) value -> . NUMBER
(8) list -> . value LIST_SEP EOL list_next
(9) list -> . value LIST_SEP list_next
(10) list -> . value LIST_SEP
TEXT shift and go to state 12
YES shift and go to state 16
NO shift and go to state 11
NUMBER shift and go to state 14
list shift and go to state 13
value shift and go to state 15
state 10
(19) section -> NAME LBRACE section_content . RBRACE
(15) section_content -> section_content . EOL
(17) section_content -> section_content . assignation
(18) section_content -> section_content . section
(2) assignation -> . NAME ASSIGN value
(3) assignation -> . NAME ASSIGN list
(19) section -> . NAME LBRACE section_content RBRACE
RBRACE shift and go to state 17
EOL shift and go to state 6
NAME shift and go to state 4
section shift and go to state 5
assignation shift and go to state 7
state 11
(6) value -> NO .
LIST_SEP reduce using rule 6 (value -> NO .)
EOL reduce using rule 6 (value -> NO .)
NAME reduce using rule 6 (value -> NO .)
$end reduce using rule 6 (value -> NO .)
RBRACE reduce using rule 6 (value -> NO .)
state 12
(4) value -> TEXT .
LIST_SEP reduce using rule 4 (value -> TEXT .)
EOL reduce using rule 4 (value -> TEXT .)
NAME reduce using rule 4 (value -> TEXT .)
$end reduce using rule 4 (value -> TEXT .)
RBRACE reduce using rule 4 (value -> TEXT .)
state 13
(3) assignation -> NAME ASSIGN list .
RBRACE reduce using rule 3 (assignation -> NAME ASSIGN list .)
EOL reduce using rule 3 (assignation -> NAME ASSIGN list .)
NAME reduce using rule 3 (assignation -> NAME ASSIGN list .)
$end reduce using rule 3 (assignation -> NAME ASSIGN list .)
state 14
(7) value -> NUMBER .
LIST_SEP reduce using rule 7 (value -> NUMBER .)
EOL reduce using rule 7 (value -> NUMBER .)
NAME reduce using rule 7 (value -> NUMBER .)
$end reduce using rule 7 (value -> NUMBER .)
RBRACE reduce using rule 7 (value -> NUMBER .)
state 15
(2) assignation -> NAME ASSIGN value .
(8) list -> value . LIST_SEP EOL list_next
(9) list -> value . LIST_SEP list_next
(10) list -> value . LIST_SEP
RBRACE reduce using rule 2 (assignation -> NAME ASSIGN value .)
EOL reduce using rule 2 (assignation -> NAME ASSIGN value .)
NAME reduce using rule 2 (assignation -> NAME ASSIGN value .)
$end reduce using rule 2 (assignation -> NAME ASSIGN value .)
LIST_SEP shift and go to state 18
state 16
(5) value -> YES .
LIST_SEP reduce using rule 5 (value -> YES .)
EOL reduce using rule 5 (value -> YES .)
NAME reduce using rule 5 (value -> YES .)
$end reduce using rule 5 (value -> YES .)
RBRACE reduce using rule 5 (value -> YES .)
state 17
(19) section -> NAME LBRACE section_content RBRACE .
RBRACE reduce using rule 19 (section -> NAME LBRACE section_content RBRACE .)
EOL reduce using rule 19 (section -> NAME LBRACE section_content RBRACE .)
NAME reduce using rule 19 (section -> NAME LBRACE section_content RBRACE .)
$end reduce using rule 19 (section -> NAME LBRACE section_content RBRACE .)
state 18
(8) list -> value LIST_SEP . EOL list_next
(9) list -> value LIST_SEP . list_next
(10) list -> value LIST_SEP .
(11) list_next -> . value LIST_SEP EOL list_next
(12) list_next -> . value LIST_SEP list_next
(13) list_next -> . value LIST_SEP
(14) list_next -> . value
(4) value -> . TEXT
(5) value -> . YES
(6) value -> . NO
(7) value -> . NUMBER
! shift/reduce conflict for EOL resolved as shift
EOL shift and go to state 20
NAME reduce using rule 10 (list -> value LIST_SEP .)
$end reduce using rule 10 (list -> value LIST_SEP .)
RBRACE reduce using rule 10 (list -> value LIST_SEP .)
TEXT shift and go to state 12
YES shift and go to state 16
NO shift and go to state 11
NUMBER shift and go to state 14
! EOL [ reduce using rule 10 (list -> value LIST_SEP .) ]
list_next shift and go to state 21
value shift and go to state 19
state 19
(11) list_next -> value . LIST_SEP EOL list_next
(12) list_next -> value . LIST_SEP list_next
(13) list_next -> value . LIST_SEP
(14) list_next -> value .
LIST_SEP shift and go to state 22
EOL reduce using rule 14 (list_next -> value .)
NAME reduce using rule 14 (list_next -> value .)
$end reduce using rule 14 (list_next -> value .)
RBRACE reduce using rule 14 (list_next -> value .)
state 20
(8) list -> value LIST_SEP EOL . list_next
(11) list_next -> . value LIST_SEP EOL list_next
(12) list_next -> . value LIST_SEP list_next
(13) list_next -> . value LIST_SEP
(14) list_next -> . value
(4) value -> . TEXT
(5) value -> . YES
(6) value -> . NO
(7) value -> . NUMBER
TEXT shift and go to state 12
YES shift and go to state 16
NO shift and go to state 11
NUMBER shift and go to state 14
list_next shift and go to state 23
value shift and go to state 19
state 21
(9) list -> value LIST_SEP list_next .
EOL reduce using rule 9 (list -> value LIST_SEP list_next .)
NAME reduce using rule 9 (list -> value LIST_SEP list_next .)
$end reduce using rule 9 (list -> value LIST_SEP list_next .)
RBRACE reduce using rule 9 (list -> value LIST_SEP list_next .)
state 22
(11) list_next -> value LIST_SEP . EOL list_next
(12) list_next -> value LIST_SEP . list_next
(13) list_next -> value LIST_SEP .
(11) list_next -> . value LIST_SEP EOL list_next
(12) list_next -> . value LIST_SEP list_next
(13) list_next -> . value LIST_SEP
(14) list_next -> . value
(4) value -> . TEXT
(5) value -> . YES
(6) value -> . NO
(7) value -> . NUMBER
! shift/reduce conflict for EOL resolved as shift
EOL shift and go to state 24
NAME reduce using rule 13 (list_next -> value LIST_SEP .)
$end reduce using rule 13 (list_next -> value LIST_SEP .)
RBRACE reduce using rule 13 (list_next -> value LIST_SEP .)
TEXT shift and go to state 12
YES shift and go to state 16
NO shift and go to state 11
NUMBER shift and go to state 14
! EOL [ reduce using rule 13 (list_next -> value LIST_SEP .) ]
list_next shift and go to state 25
value shift and go to state 19
state 23
(8) list -> value LIST_SEP EOL list_next .
EOL reduce using rule 8 (list -> value LIST_SEP EOL list_next .)
NAME reduce using rule 8 (list -> value LIST_SEP EOL list_next .)
$end reduce using rule 8 (list -> value LIST_SEP EOL list_next .)
RBRACE reduce using rule 8 (list -> value LIST_SEP EOL list_next .)
state 24
(11) list_next -> value LIST_SEP EOL . list_next
(11) list_next -> . value LIST_SEP EOL list_next
(12) list_next -> . value LIST_SEP list_next
(13) list_next -> . value LIST_SEP
(14) list_next -> . value
(4) value -> . TEXT
(5) value -> . YES
(6) value -> . NO
(7) value -> . NUMBER
TEXT shift and go to state 12
YES shift and go to state 16
NO shift and go to state 11
NUMBER shift and go to state 14
list_next shift and go to state 26
value shift and go to state 19
state 25
(12) list_next -> value LIST_SEP list_next .
EOL reduce using rule 12 (list_next -> value LIST_SEP list_next .)
NAME reduce using rule 12 (list_next -> value LIST_SEP list_next .)
$end reduce using rule 12 (list_next -> value LIST_SEP list_next .)
RBRACE reduce using rule 12 (list_next -> value LIST_SEP list_next .)
state 26
(11) list_next -> value LIST_SEP EOL list_next .
EOL reduce using rule 11 (list_next -> value LIST_SEP EOL list_next .)
NAME reduce using rule 11 (list_next -> value LIST_SEP EOL list_next .)
$end reduce using rule 11 (list_next -> value LIST_SEP EOL list_next .)
RBRACE reduce using rule 11 (list_next -> value LIST_SEP EOL list_next .)
WARNING:
WARNING: Conflicts:
WARNING:
WARNING: shift/reduce conflict for EOL in state 18 resolved as shift
WARNING: shift/reduce conflict for EOL in state 22 resolved as shift
Raw
parser.py
""" Dotconf configuration parser.
"""
import ply.lex as lex
import ply.yacc as yacc
from dotconf.section import Section
#
# Lexer
#
class DotconfLexer(object):
""" Lexer for the DotConf format.
:param **kwargs: arguments to give to the ply lexer
Usage example::
>>> lexer = DotConfLexer()
>>> lexer.input('test { key = yes }')
>>> print lexer.next()
"""
def __init__(self, **kwargs):
self._lexer = lex.lex(module=self, **kwargs)
#
# Tokens definition
#
reserved = {'yes': 'YES', 'no': 'NO'}
tokens = ['LBRACE', 'RBRACE', 'NAME', 'TEXT', 'NUMBER',
'ASSIGN', 'EOL', 'LIST_SEP'] + reserved.values()
t_LBRACE = '{'
t_RBRACE = '}'
t_ASSIGN = '='
t_LIST_SEP = ','
t_ignore = ' \t'
def t_NAME(self, token):
r'[a-zA-Z_][a-zA-Z0-9_-]*'
token.type = self.reserved.get(token.value, 'NAME')
# Handle the boolean case:
if token.type == 'YES':
token.value = True
elif token.type == 'NO':
token.value = False
return token
def t_TEXT(self, token):
r'(["]([\\]["]|[^"]|)*["]|[\']([\\][\']|[^\'])*[\'])'
value = token.value[1:-1].replace('\\' + token.value[0], token.value[0])
token.value = value
# Count the lines in the string:
token.lexer.lineno += value.count('\n')
return token
def t_NUMBER(self, token):
r'[-+]?[0-9]+(\.[0-9]+)?'
if token.value.isdigit():
token.value = int(token.value)
else:
token.value = float(token.value)
return token
def t_EOL(self, token):
r'[\n]+'
token.lexer.lineno += len(token.value)
return token
#
# Bindings to the internal _lexer object
#
def __getattr__(self, name):
attr = getattr(self._lexer, name)
if attr is None:
raise AttributeError("'%s' object has no attribute '%s'" % (self, name))
else:
return attr
#
# Parser
#
class DotconfParser(object):
""" Parser for the Dotconf format.
"""
tokens = DotconfLexer.tokens
def __init__(self, **kwargs):
self._parser = yacc.yacc(module=self, **kwargs)
#
# Rules
#
start = 'top'
def p_top(self, p):
"""top : section_content"""
section = Section('__top__')
for name, child, is_section in p[1]:
if is_section:
child.parent = section
section.register_child(name, child)
p[0] = section
def p_assignation(self, p):
"""assignation : NAME ASSIGN value
| NAME ASSIGN list"""
p[0] = {'name': p[1],
'value': p[3]}
def p_value(self, p):
"""value : TEXT
| YES
| NO
| NUMBER"""
p[0] = p[1]
def p_list_multiline(self, p):
"""list : value LIST_SEP EOL list_next"""
p[4].insert(0, p[1])
p[0] = p[4]
def p_list(self, p):
"""list : value LIST_SEP list_next"""
p[3].insert(0, p[1])
p[0] = p[3]
def p_list_singleton(self, p):
"""list : value LIST_SEP"""
p[0] = [p[1]]
def p_list_next_multiline(self, p):
"""list_next : value LIST_SEP EOL list_next"""
p[4].insert(0, p[1])
p[0] = p[4]
def p_list_next(self, p):
"""list_next : value LIST_SEP list_next"""
p[3].insert(0, p[1])
p[0] = p[3]
def p_list_next_last(self, p):
"""list_next : value LIST_SEP
| value"""
p[0] = [p[1]]
def p_section_content(self, p):
"""section_content : section_content EOL"""
p[0] = p[1]
def p_section_content_empty(self, p):
"""section_content : empty"""
p[0] = []
def p_section_content_assignation(self, p):
"""section_content : section_content assignation"""
p[1].append((p[2]['name'], p[2]['value'], False))
p[0] = p[1]
def p_section_content_section(self, p):
"""section_content : section_content section"""
p[1].append((p[2].name, p[2], True))
p[0] = p[1]
def p_section(self, p):
"""section : NAME LBRACE section_content RBRACE"""
section = Section(p[1])
for name, child, is_section in p[3]:
if is_section:
child.parent = section
section.register_child(name, child)
p[0] = section
def p_empty(self, p):
"""empty :"""
pass
#def p_error(self, p):
# print p
#
# Bindings to the internel _parser object
#
def __getattr__(self, name):
attr = getattr(self._parser, name)
if attr is None:
raise AttributeError("'%s' object has no attribute '%s'" % (self, name))
else:
return attr
#
# Tests
#
from nose.tools import eq_
def test_lexer():
# Test string Expected tok Expected tok value
tokens = (('name', 'NAME', 'name'),
('"test"', 'TEXT', 'test'),
("'test'", 'TEXT', 'test'),
(r"'te\'st'", 'TEXT', "te'st"),
('42', 'NUMBER', 42),
('42.1', 'NUMBER', 42.1),
('+42', 'NUMBER', 42),
('+42.1', 'NUMBER', 42.1),
('-42', 'NUMBER', -42),
('-42.1', 'NUMBER', -42.1),
('{', 'LBRACE', '{'),
('}', 'RBRACE', '}'),
('=', 'ASSIGN', '='))
for test, expected_type, expected_value in tokens:
yield check_token, test, expected_type, expected_value
def check_token(test, expected_type, expected_value):
lexer = DotconfLexer()
lexer.input(test)
token = lexer.next()
eq_(token.type, expected_type)
eq_(token.value, expected_value)
def test_parser():
test = '''
daemon = yes
lol = 42
list = 1
list2_ = 1,2,3,4
test {
tust = 42
section {
test = "lol :D
mdr
"
}
section {
lul = no
}
}
'''
lexer = DotconfLexer()
#lexer.input(test)
#list(lexer._lexer)
parser = DotconfParser(debug=True, write_tables=False, errorlog=yacc.NullLogger())
output = parser.parse(test, lexer=lexer, tracking=True)
import pprint
pprint.pprint(output)
return output
#eq_(output, 'test')
if __name__ == '__main__':
out = test_parser()
print out.get('list2_')
Raw
section.py
""" Classes used to represent the configuration tree
"""
class Section(object):
""" Represent a (sub-)section of the configuration.
"""
def __init__(self, name, parent=None):
self._name = name
self._parent = parent
self._children = {} # All children of this section stored as a dict
# key -> list of children for the key.
def __repr__(self):
return "<Section '%s' with %d children>" % (self.name,
len(self._children))
#
# Public API -- Tree construction methods
# These methods are intended as internal usage for the configuration
# tree construction by the parser.
#
def register_child(self, name, child):
key_children = self._children.get(name, [])
key_children.append(child)
self._children[name] = key_children
#
# Public API -- User methods
# These methods are intended for configuration tree traversal for
# user usage.
#
@property
def name(self):
return self._name
@property
def parent(self):
return self._parent
@parent.setter
def parent(self, parent):
self._parent = parent
def subsections(self, name):
""" Iterate over sub-sections with the specified name.
"""
for children in self._children.get(name, ()):
if isinstance(children, Section):
yield children
def get(self, name, default=None):
""" Get the last value found with the specified name or
return default.
"""
found = False
child = None
for child in self._children.get(name, ()):
if not isinstance(child, Section):
found = True
if found:
return child
else:
return default
def getiter(self, name):
""" Iterate over all values with the specified name.
"""
for children in self._children.get(name, ()):
if not isinstance(children, Section):
yield children
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment