Skip to content

Instantly share code, notes, and snippets.

@zurk

zurk/tree_visualization.ipynb

Last active September 27, 2017 15:36
Show Gist options
  • Save zurk/d314d67d9aac8843d3776c82cd738b40 to your computer and use it in GitHub Desktop.
Save zurk/d314d67d9aac8843d3776c82cd738b40 to your computer and use it in GitHub Desktop.
Download ZIP
Babelfish tree text visualization for roles structure.ipynb
Display the source blob
Display the rendered blob
Raw
tree_visualization.ipynb
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Bblfsh role tree visualization"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"collapsed": true,
"scrolled": false
},
"outputs": [],
"source": [
"import bblfsh\n",
"from bblfsh import BblfshClient\n",
"import importlib\n",
"DESCRIPTOR = importlib.import_module(\n",
" \"bblfsh.gopkg.in.bblfsh.sdk.%s.uast.generated_pb2\" % bblfsh.sdkversion.VERSION).DESCRIPTOR\n",
"Node = importlib.import_module(\n",
" \"bblfsh.gopkg.in.bblfsh.sdk.%s.uast.generated_pb2\" % bblfsh.sdkversion.VERSION).Node\n",
"\n",
"def role_id(role_name):\n",
" return DESCRIPTOR.enum_types_by_name[\"Role\"].values_by_name[role_name].number\n",
"\n",
"def role_name(role_id):\n",
" return DESCRIPTOR.enum_types_by_name[\"Role\"].values_by_number[role_id].name\n",
"\n",
"def node2raw(node, internal_type=True):\n",
" token = \"|%s|\" % node.token[:20].replace(\"\\n\", \"\\\\n\")\n",
" roles = \", \".join([role_name(k) for k in node.roles])\n",
" internal_role = node.internal_type\n",
" positions = str(node.start_position.line) if node.start_position.line != 0 else \"\"\n",
" return [positions, token, internal_role, roles] if internal_type else [positions, token, roles]\n",
" \n",
"def uast_roles(uast) -> iter:\n",
" stack = [(0, uast)]\n",
" while stack:\n",
" level, node = stack.pop()\n",
" yield level, node\n",
" stack.extend((level+1, c) for c in node.children[::-1])\n",
"\n",
"def rreplace(s, old, new, times):\n",
" li = s.rsplit(old, times)\n",
" return new.join(li)\n",
" \n",
"def print_uast_structure(uast, internal_type=True):\n",
" if internal_type:\n",
" table = [[\"#\", \"Token\", \"Internal Role\", \"Roles Tree\"], [\"\", \"\", \"\", \"\"]]\n",
" old_level = 0\n",
" for level, node in uast_roles(uast):\n",
" raw = node2raw(node, internal_type)\n",
" if level != 0:\n",
" raw[-1] = \"┃ \" * (level-1) + \"┣ \" + raw[-1]\n",
" if level < old_level:\n",
" table[-1][-1] = rreplace(table[-1][-1].replace(\"┣\", \"┗\"), \"┃\", \"┗\", old_level-level-1)\n",
" pass\n",
" table.append(raw)\n",
" old_level = level\n",
" table[-1][-1] = table[-1][-1].replace(\"┣\", \"┗\").replace(\"┃\", \"┗\")\n",
"\n",
" col_size = [max(len(cell) for cell in table_col) for table_col in zip(*table)]\n",
" for raw in table:\n",
" print(\" \".join(cell.ljust(size) for cell, size in zip(raw, col_size)))"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"import lib1\n",
"import lib2.lib21\n",
"import lib3 as lib3_alias\n",
"from lib4 import lib41\n",
"from lib5.lib51 import lib511\n",
"from lib6 import lib61 as lib611_alias\n",
"\n",
"var1 = None\n",
"var2 = None\n",
"\n",
"class class1:\n",
"\tvar11 = None\n",
"\n",
"\tdef __init__(self):\n",
"\t\tlib1.lib11.lib111.var1111 = None\n",
"\t\tself.var101 = None\n",
"\t\tvar102, var103 = None, None\n",
"\n",
"\tdef func11(self):\n",
"\t\tclass1.var111 = None\n",
"\t\tself.var112 = None\n",
"\t\tvar113 = None\n",
"\n",
"\tdef func12():\n",
"\t\tdef func121():\n",
"\t\t\tpass\n",
"\n",
"class class2:\n",
"\tvar21 = None\n",
"\tdef func21(self):\n",
"\t\tvar21 < 0\n",
"\t\tvar21 == 0\n",
"\tdef func22(self, arg1):\n",
"\t\tNone\n",
"\tdef func23():\n",
"\t\tNone\n",
"\n",
"class class3:\n",
"\tpass\n",
"\n"
]
}
],
"source": [
"filepath = \"./example.py\"\n",
"with open(filepath) as f:\n",
" print(f.read())"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"scrolled": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"# Token Internal Role Roles Tree \n",
" \n",
" || Module FILE \n",
"1 || Import ┣ IMPORT, DECLARATION, STATEMENT \n",
" |lib1| alias ┃ ┗ IMPORT, PATHNAME, IDENTIFIER \n",
"2 || Import ┣ IMPORT, DECLARATION, STATEMENT \n",
" |lib2.lib21| alias ┃ ┗ IMPORT, PATHNAME, IDENTIFIER \n",
"3 || Import ┣ IMPORT, DECLARATION, STATEMENT \n",
" |lib3| alias ┃ ┣ IMPORT, PATHNAME, IDENTIFIER \n",
" |lib3_alias| alias.asname ┃ ┗ ┗ IMPORT, ALIAS, IDENTIFIER \n",
"4 || ImportFrom ┣ IMPORT, DECLARATION, STATEMENT \n",
" |lib4| ImportFrom.module ┃ ┣ IMPORT, PATHNAME, IDENTIFIER \n",
" |lib41| alias ┃ ┗ IMPORT, PATHNAME, IDENTIFIER \n",
"5 || ImportFrom ┣ IMPORT, DECLARATION, STATEMENT \n",
" |lib5.lib51| ImportFrom.module ┃ ┣ IMPORT, PATHNAME, IDENTIFIER \n",
" |lib511| alias ┃ ┗ IMPORT, PATHNAME, IDENTIFIER \n",
"6 || ImportFrom ┣ IMPORT, DECLARATION, STATEMENT \n",
" |lib6| ImportFrom.module ┃ ┣ IMPORT, PATHNAME, IDENTIFIER \n",
" |lib61| alias ┃ ┣ IMPORT, PATHNAME, IDENTIFIER \n",
" |lib611_alias| alias.asname ┃ ┗ ┗ IMPORT, ALIAS, IDENTIFIER \n",
"8 || Assign ┣ BINARY, THIS, EXPRESSION \n",
"8 |var1| Name ┃ ┣ LEFT, IDENTIFIER, EXPRESSION \n",
"7 || PreviousNoops ┃ ┃ ┣ INCOMPLETE \n",
"7 |\\n| NoopLine ┃ ┃ ┗ ┗ DOCUMENTATION \n",
"8 |<nil>| NoneLiteral ┃ ┗ BYTE, NUMBER, EXPRESSION, RIGHT \n",
"9 || Assign ┣ BINARY, THIS, EXPRESSION \n",
"9 |var2| Name ┃ ┣ LEFT, IDENTIFIER, EXPRESSION \n",
"9 |<nil>| NoneLiteral ┃ ┗ BYTE, NUMBER, EXPRESSION, RIGHT \n",
"11 |class1| ClassDef ┣ ENTRY, DECLARATION, IDENTIFIER, STATEMENT \n",
" || ClassDef.body ┃ ┣ ENTRY, DECLARATION, BODY \n",
"12 || Assign ┃ ┃ ┣ BINARY, THIS, EXPRESSION \n",
"12 |var11| Name ┃ ┃ ┃ ┣ LEFT, IDENTIFIER, EXPRESSION \n",
"10 || PreviousNoops ┃ ┃ ┃ ┃ ┣ INCOMPLETE \n",
"10 |\\n| NoopLine ┃ ┃ ┃ ┃ ┗ ┗ DOCUMENTATION \n",
"12 |<nil>| NoneLiteral ┃ ┃ ┃ ┗ BYTE, NUMBER, EXPRESSION, RIGHT \n",
"14 |__init__| FunctionDef ┃ ┃ ┣ FUNCTION, DECLARATION, NAME, IDENTIFIER \n",
" || arguments ┃ ┃ ┃ ┣ FUNCTION, DECLARATION, ARGUMENT, NAME, IDENTIFIER \n",
"14 |self| arg ┃ ┃ ┃ ┃ ┣ FUNCTION, DECLARATION, ARGUMENT, NAME, IDENTIFIER \n",
"13 || PreviousNoops ┃ ┃ ┃ ┃ ┃ ┣ INCOMPLETE \n",
"13 |\\n| NoopLine ┃ ┃ ┃ ┃ ┗ ┗ ┗ DOCUMENTATION \n",
" || FunctionDef.body ┃ ┃ ┃ ┣ FUNCTION, DECLARATION, BODY \n",
"15 || Assign ┃ ┃ ┃ ┃ ┣ BINARY, THIS, EXPRESSION \n",
"15 |var1111| Attribute ┃ ┃ ┃ ┃ ┃ ┣ IDENTIFIER, EXPRESSION, LEFT \n",
"15 |lib111| Attribute ┃ ┃ ┃ ┃ ┃ ┃ ┣ IDENTIFIER, EXPRESSION \n",
"15 |lib11| Attribute ┃ ┃ ┃ ┃ ┃ ┃ ┃ ┣ IDENTIFIER, EXPRESSION \n",
"15 |lib1| Name ┃ ┃ ┃ ┃ ┃ ┃ ┗ ┗ ┗ IDENTIFIER, QUALIFIED, IDENTIFIER, EXPRESSION\n",
"15 |<nil>| NoneLiteral ┃ ┃ ┃ ┃ ┃ ┗ BYTE, NUMBER, EXPRESSION, RIGHT \n",
"16 || Assign ┃ ┃ ┃ ┃ ┣ BINARY, THIS, EXPRESSION \n",
"16 |var101| Attribute ┃ ┃ ┃ ┃ ┃ ┣ IDENTIFIER, EXPRESSION, LEFT \n",
"16 |self| Name ┃ ┃ ┃ ┃ ┃ ┃ ┗ IDENTIFIER, QUALIFIED, IDENTIFIER, EXPRESSION \n",
"16 |<nil>| NoneLiteral ┃ ┃ ┃ ┃ ┃ ┗ BYTE, NUMBER, EXPRESSION, RIGHT \n",
"17 || Assign ┃ ┃ ┃ ┃ ┣ BINARY, THIS, EXPRESSION \n",
"17 || Tuple ┃ ┃ ┃ ┃ ┃ ┣ BYTE, TYPE, EXPRESSION, LEFT \n",
"17 |var102| Name ┃ ┃ ┃ ┃ ┃ ┃ ┣ IDENTIFIER, EXPRESSION \n",
"17 |var103| Name ┃ ┃ ┃ ┃ ┃ ┃ ┗ IDENTIFIER, EXPRESSION \n",
"17 || Tuple ┃ ┃ ┃ ┃ ┃ ┣ BYTE, TYPE, EXPRESSION, RIGHT \n",
"17 |<nil>| NoneLiteral ┃ ┃ ┃ ┃ ┃ ┃ ┣ BYTE, NUMBER, EXPRESSION \n",
"17 |<nil>| NoneLiteral ┃ ┃ ┃ ┗ ┗ ┗ ┗ BYTE, NUMBER, EXPRESSION \n",
"19 |func11| FunctionDef ┃ ┃ ┣ FUNCTION, DECLARATION, NAME, IDENTIFIER \n",
" || arguments ┃ ┃ ┃ ┣ FUNCTION, DECLARATION, ARGUMENT, NAME, IDENTIFIER \n",
"19 |self| arg ┃ ┃ ┃ ┃ ┣ FUNCTION, DECLARATION, ARGUMENT, NAME, IDENTIFIER \n",
"18 || PreviousNoops ┃ ┃ ┃ ┃ ┃ ┣ INCOMPLETE \n",
"18 |\\n| NoopLine ┃ ┃ ┃ ┃ ┗ ┗ ┗ DOCUMENTATION \n",
" || FunctionDef.body ┃ ┃ ┃ ┣ FUNCTION, DECLARATION, BODY \n",
"20 || Assign ┃ ┃ ┃ ┃ ┣ BINARY, THIS, EXPRESSION \n",
"20 |var111| Attribute ┃ ┃ ┃ ┃ ┃ ┣ IDENTIFIER, EXPRESSION, LEFT \n",
"20 |class1| Name ┃ ┃ ┃ ┃ ┃ ┃ ┗ IDENTIFIER, QUALIFIED, IDENTIFIER, EXPRESSION \n",
"20 |<nil>| NoneLiteral ┃ ┃ ┃ ┃ ┃ ┗ BYTE, NUMBER, EXPRESSION, RIGHT \n",
"21 || Assign ┃ ┃ ┃ ┃ ┣ BINARY, THIS, EXPRESSION \n",
"21 |var112| Attribute ┃ ┃ ┃ ┃ ┃ ┣ IDENTIFIER, EXPRESSION, LEFT \n",
"21 |self| Name ┃ ┃ ┃ ┃ ┃ ┃ ┗ IDENTIFIER, QUALIFIED, IDENTIFIER, EXPRESSION \n",
"21 |<nil>| NoneLiteral ┃ ┃ ┃ ┃ ┃ ┗ BYTE, NUMBER, EXPRESSION, RIGHT \n",
"22 || Assign ┃ ┃ ┃ ┃ ┣ BINARY, THIS, EXPRESSION \n",
"22 |var113| Name ┃ ┃ ┃ ┃ ┃ ┣ LEFT, IDENTIFIER, EXPRESSION \n",
"22 |<nil>| NoneLiteral ┃ ┃ ┃ ┗ ┗ ┗ BYTE, NUMBER, EXPRESSION, RIGHT \n",
"24 |func12| FunctionDef ┃ ┃ ┣ FUNCTION, DECLARATION, NAME, IDENTIFIER \n",
" || FunctionDef.body ┃ ┃ ┃ ┣ FUNCTION, DECLARATION, BODY \n",
"25 |func121| FunctionDef ┃ ┃ ┃ ┃ ┣ FUNCTION, DECLARATION, NAME, IDENTIFIER \n",
" || FunctionDef.body ┃ ┃ ┃ ┃ ┃ ┣ FUNCTION, DECLARATION, BODY \n",
"26 |pass| Pass ┃ ┃ ┃ ┃ ┃ ┃ ┣ LITERAL, STATEMENT \n",
"23 || PreviousNoops ┃ ┃ ┃ ┃ ┃ ┃ ┃ ┣ INCOMPLETE \n",
"23 |\\n| NoopLine ┃ ┃ ┃ ┃ ┃ ┃ ┗ ┗ ┗ DOCUMENTATION \n",
" || arguments ┃ ┃ ┃ ┃ ┗ ┗ FUNCTION, DECLARATION, ARGUMENT, NAME, IDENTIFIER \n",
" || arguments ┃ ┗ ┗ ┗ FUNCTION, DECLARATION, ARGUMENT, NAME, IDENTIFIER \n",
"28 |class2| ClassDef ┣ ENTRY, DECLARATION, IDENTIFIER, STATEMENT \n",
" || ClassDef.body ┃ ┣ ENTRY, DECLARATION, BODY \n",
"29 || Assign ┃ ┃ ┣ BINARY, THIS, EXPRESSION \n",
"29 |var21| Name ┃ ┃ ┃ ┣ LEFT, IDENTIFIER, EXPRESSION \n",
"27 || PreviousNoops ┃ ┃ ┃ ┃ ┣ INCOMPLETE \n",
"27 |\\n| NoopLine ┃ ┃ ┃ ┃ ┗ ┗ DOCUMENTATION \n",
"29 |<nil>| NoneLiteral ┃ ┃ ┃ ┗ BYTE, NUMBER, EXPRESSION, RIGHT \n",
"30 |func21| FunctionDef ┃ ┃ ┣ FUNCTION, DECLARATION, NAME, IDENTIFIER \n",
" || arguments ┃ ┃ ┃ ┣ FUNCTION, DECLARATION, ARGUMENT, NAME, IDENTIFIER \n",
"30 |self| arg ┃ ┃ ┃ ┃ ┗ FUNCTION, DECLARATION, ARGUMENT, NAME, IDENTIFIER \n",
" || FunctionDef.body ┃ ┃ ┃ ┣ FUNCTION, DECLARATION, BODY \n",
"31 || Expr ┃ ┃ ┃ ┃ ┣ EXPRESSION \n",
"31 || Compare ┃ ┃ ┃ ┃ ┃ ┣ EXPRESSION, BINARY \n",
" || Compare.comparators ┃ ┃ ┃ ┃ ┃ ┃ ┣ EXPRESSION, BINARY, RIGHT \n",
"31 |0| Num ┃ ┃ ┃ ┃ ┃ ┃ ┃ ┗ BYTE, REGEXP, EXPRESSION \n",
"31 |var21| Name ┃ ┃ ┃ ┃ ┃ ┃ ┣ IDENTIFIER, EXPRESSION, EXPRESSION, BINARY, LEFT \n",
" || Compare.ops ┃ ┃ ┃ ┃ ┃ ┃ ┣ EXPRESSION, BINARY, OPERATOR \n",
" |<| Lt ┃ ┃ ┃ ┃ ┃ ┗ ┗ ┗ BINARY, OPERATOR, LESS_THAN \n",
"32 || Expr ┃ ┃ ┃ ┃ ┣ EXPRESSION \n",
"32 || Compare ┃ ┃ ┃ ┃ ┃ ┣ EXPRESSION, BINARY \n",
" || Compare.comparators ┃ ┃ ┃ ┃ ┃ ┃ ┣ EXPRESSION, BINARY, RIGHT \n",
"32 |0| Num ┃ ┃ ┃ ┃ ┃ ┃ ┃ ┗ BYTE, REGEXP, EXPRESSION \n",
"32 |var21| Name ┃ ┃ ┃ ┃ ┃ ┃ ┣ IDENTIFIER, EXPRESSION, EXPRESSION, BINARY, LEFT \n",
" || Compare.ops ┃ ┃ ┃ ┃ ┃ ┃ ┣ EXPRESSION, BINARY, OPERATOR \n",
" |==| Eq ┃ ┃ ┃ ┗ ┗ ┗ ┗ ┗ BINARY, OPERATOR, EQUAL \n",
"33 |func22| FunctionDef ┃ ┃ ┣ FUNCTION, DECLARATION, NAME, IDENTIFIER \n",
" || arguments ┃ ┃ ┃ ┣ FUNCTION, DECLARATION, ARGUMENT, NAME, IDENTIFIER \n",
"33 |self| arg ┃ ┃ ┃ ┃ ┣ FUNCTION, DECLARATION, ARGUMENT, NAME, IDENTIFIER \n",
"33 |arg1| arg ┃ ┃ ┃ ┃ ┗ FUNCTION, DECLARATION, ARGUMENT, NAME, IDENTIFIER \n",
" || FunctionDef.body ┃ ┃ ┃ ┣ FUNCTION, DECLARATION, BODY \n",
"34 || Expr ┃ ┃ ┃ ┃ ┣ EXPRESSION \n",
"34 |<nil>| NoneLiteral ┃ ┃ ┃ ┗ ┗ ┗ BYTE, NUMBER, EXPRESSION \n",
"35 |func23| FunctionDef ┃ ┃ ┣ FUNCTION, DECLARATION, NAME, IDENTIFIER \n",
" || FunctionDef.body ┃ ┃ ┃ ┣ FUNCTION, DECLARATION, BODY \n",
"36 || Expr ┃ ┃ ┃ ┃ ┣ EXPRESSION \n",
"36 |<nil>| NoneLiteral ┃ ┃ ┃ ┃ ┗ ┗ BYTE, NUMBER, EXPRESSION \n",
" || arguments ┃ ┗ ┗ ┗ FUNCTION, DECLARATION, ARGUMENT, NAME, IDENTIFIER \n",
"38 |class3| ClassDef ┣ ENTRY, DECLARATION, IDENTIFIER, STATEMENT \n",
" || ClassDef.body ┃ ┣ ENTRY, DECLARATION, BODY \n",
"39 |pass| Pass ┃ ┃ ┣ LITERAL, STATEMENT \n",
"37 || PreviousNoops ┃ ┃ ┃ ┣ INCOMPLETE \n",
"37 |\\n| NoopLine ┗ ┗ ┗ ┗ ┗ DOCUMENTATION \n"
]
}
],
"source": [
"client = BblfshClient(\"0.0.0.0:9432\")\n",
"\n",
"response = client.parse(filepath)\n",
"uast = response.uast\n",
"print_uast_structure(uast)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.1"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment