Skip to content

Instantly share code, notes, and snippets.

@pallada-92

pallada-92/160826_gchatstats_backend.ipynb

Created September 26, 2016 16:49
Show Gist options
  • Save pallada-92/fce7a5e1ad5f895b8c698061f30764be to your computer and use it in GitHub Desktop.
Save pallada-92/fce7a5e1ad5f895b8c698061f30764be to your computer and use it in GitHub Desktop.
Download ZIP
GChatStats
Display the source blob
Display the rendered blob
Raw
160826_gchatstats_backend.ipynb
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Display the source blob
Display the rendered blob
Raw
160826_gchatstats_upload.ipynb
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Display the source blob
Display the rendered blob
Raw
160826_gchatstats_view_report.ipynb
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Display the source blob
Display the rendered blob
Raw
160907_jupyter_ext.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"%%requirejs /static/requirejs@2.3.1/require.min.js\n",
"$ jquery /static/jquery@2.1.1/jquery.min.js\n",
"React react /static/react@15.3.1/build/react.js\n",
"ReactDOM /static/react@15.3.1/build/react-dom.min.js\n",
"R /static/ramda@0.22.1/ramda.min.js\n",
"d3 /static/d3@4.2.2/d3.min.js\n",
"snap /static/snapsvg@0.4.1/dist/snap.svg-min.js\n",
"vis /static/visjs@4.16.1/dist/vis.min.js"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"import nbformat\n",
"import extensions.my_ext\n",
"\n",
"# import importlib\n",
"# importlib.reload(extensions.my_ext)\n",
"\n",
"extensions.my_ext.load_ipython_extension(get_ipython())\n",
"\n",
"def make(fname):\n",
" source = open(fname, 'rb').read()\n",
" nb = nbformat.reads(source.decode('utf8'), as_version=4)\n",
" parents = {}\n",
" cells = [extensions.my_ext.NotebookCell(cell, parents, index) for index, cell in enumerate(nb.cells)]\n",
" with open('/data/sets/static/p/gchatstats/upload.html', 'w') as f:\n",
" f.write(make_html(cells))\n",
"\n",
"def make_html(cells):\n",
" header_html = ''\n",
" body_html = ''\n",
" jsx_scripts = ''\n",
" scripts = ''\n",
" run_scripts = ''\n",
" for cell in cells:\n",
" if cell.magic == '%%js' and '@out' in cell.labels:\n",
" scripts += cell.get_output('application/javascript') + '\\n'\n",
" if cell.magic == '%%js' and '@out_run' in cell.labels:\n",
" run_scripts += cell.get_output('application/javascript') + '\\n'\n",
" elif cell.magic == '%%jsx' and '@out' in cell.labels:\n",
" jsx_scripts += cell.get_output('application/javascript') + '\\n'\n",
" elif cell.magic == '%%css' and '@out' in cell.labels:\n",
" header_html += cell.get_output('text/html') + '\\n'\n",
" elif cell.magic == '%%html' and '@header' in cell.labels:\n",
" header_html += cell.source + '\\n'\n",
" elif cell.magic == '%%html' and '@body' in cell.labels:\n",
" body_html += cell.source + '\\n'\n",
" elif cell.kind == 'code' and cell.source.startswith('%css '):\n",
" header_html += cell.get_output('text/html') + '\\n'\n",
" elif cell.magic == '%%requirejs':\n",
" require_config = cell.get_output('application/javascript') + '\\n'\n",
" res = '<!DOCTYPE html><html><head>\\n'\n",
" res += header_html + '\\n'\n",
" if hasattr(extensions.my_ext, 'requirejs'):\n",
" res += '<script src=\\\"%s\\\"></script>\\n' % extensions.my_ext.requirejs['lib_path']\n",
" res += '<script>\\n' + require_config + '\\n</script>\\n'\n",
" res += '<script>\\n' + scripts + '\\n</script>\\n'\n",
" res += '<script>\\n' + jsx_scripts + '\\n</script>\\n'\n",
" res += '</head><body>\\n'\n",
" res += body_html\n",
" res += '<script>\\n' + run_scripts + '\\n</script>\\n'\n",
" res += '</body></html>\\n'\n",
" return res"
]
},
{
"cell_type": "code",
"execution_count": 31,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Writing /data/sets/venv/jupyter/share/jupyter/nbextensions/jupyter_extension/jupyter_extension.yaml\n"
]
}
],
"source": [
"%%file /data/sets/venv/jupyter/share/jupyter/nbextensions/jupyter_extension/jupyter_extension.yaml\n",
"Type: IPython Notebook Extension\n",
"Name: Custom jupyter extension\n",
"Main: main.js\n",
"Compatibility: 4.x"
]
},
{
"cell_type": "code",
"execution_count": 76,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Overwriting /data/sets/venv/jupyter/share/jupyter/nbextensions/jupyter_extension/main.js\n"
]
}
],
"source": [
"%%file /data/sets/venv/jupyter/share/jupyter/nbextensions/jupyter_extension/main.js\n",
"define(['base/js/namespace'], function(Jupyter){\n",
" function modify_magic() {\n",
" var modes = Jupyter.CodeCell.options_default.highlight_modes;\n",
" modes.magic_javascript = {reg: [\"^%%javascript\", \"^%%js\", \"^%%jsx\"]};\n",
" modes.magic_css = {reg: [\"^%%css\"]};\n",
" modes.magic_python = {reg: ['^%%python3?', '^%%py']};\n",
" modes.magic_html = {reg: ['^%%html']};\n",
" }\n",
" function register_output_raw_cells() {\n",
" var mime = ['text/html', 'application/javascript'];\n",
" var handler = function () {\n",
" var cur_cell = Jupyter.notebook.get_selected_cell();\n",
" if (!('output_area' in cur_cell)) {\n",
" return;\n",
" }\n",
" var cell_outputs = cur_cell.output_area.outputs;\n",
" for (var i=0; i<cell_outputs.length; i++) {\n",
" var data = cell_outputs[i].data;\n",
" if (typeof data != 'object') continue;\n",
" for (var j=0; j<mime.length; j++) {\n",
" if (mime[j] in data) {\n",
" Jupyter.pager.clear();\n",
" Jupyter.pager.expand();\n",
" Jupyter.pager.append_text(mime[j]);\n",
" Jupyter.pager.append_text(data[mime[j]]);\n",
" break;\n",
" }\n",
" }\n",
" if (j == mime.length) {\n",
" var res = []\n",
" for (var m in data) {\n",
" res.push(m);\n",
" }\n",
" console.log(res);\n",
" }\n",
" }\n",
" };\n",
" var action = {\n",
" icon: 'fa-code',\n",
" help : 'inspect output source',\n",
" help_index : 'zz',\n",
" handler : handler\n",
" };\n",
" var prefix = 'auto';\n",
" var action_name = 'inspect-output-source';\n",
" var full_action_name = Jupyter.actions.register(action, action_name, prefix);\n",
" Jupyter.keyboard_manager.command_shortcuts.add_shortcut('Alt-O', full_action_name); \n",
" }\n",
" function bind_checkpoint() {\n",
" if (!window.checkpoint_bound) {\n",
" window.checkpoint_bound = true;\n",
" Jupyter.notebook.events.on('checkpoint_created.Notebook', () => {on_checkpoint()});\n",
" console.log('binding checkpoint');\n",
" }\n",
" window.on_checkpoint = function () {\n",
" IPython.notebook.kernel.execute(\n",
" 'if \"make\" in globals():\\n\\tmake(\"' + IPython.notebook.notebook_path + '\")',\n",
" {iopub: {output: function(obj) {\n",
" if (obj.msg_type == 'error') {\n",
" console.log(obj);\n",
" alert(obj.content.ename + ': ' + obj.content.evalue);\n",
" }\n",
" }}}\n",
" );\n",
" }\n",
" }\n",
" function bind_refresh_highlight() {\n",
" /*\n",
" if (!window.refresh_highlight_bound) {\n",
" window.refresh_highlight_bound = true;\n",
" Jupyter.notebook.events.on('notebook_loaded.Notebook', () => {refresh_highlight()});\n",
" console.log('binding checkpoint');\n",
" }\n",
" */\n",
" window.refresh_highlight = function () {\n",
" console.log(\"refreshing highlight\")\n",
" var cells = IPython.notebook.get_cells();\n",
" for (var i=0; i<cells.length; i++) {\n",
" cells[i].force_highlight();\n",
" }\n",
" }\n",
" refresh_highlight();\n",
" }\n",
" function load_ipython_extension() {\n",
" modify_magic();\n",
" register_output_raw_cells();\n",
" bind_checkpoint();\n",
" bind_refresh_highlight();\n",
" }\n",
" return {\n",
" load_ipython_extension: load_ipython_extension\n",
" };\n",
"});"
]
},
{
"cell_type": "code",
"execution_count": 69,
"metadata": {
"collapsed": false,
"scrolled": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"The my_ext extension is already loaded. To reload it, use:\n",
" %reload_ext my_ext\n"
]
}
],
"source": [
"%load_ext my_ext"
]
},
{
"cell_type": "code",
"execution_count": 86,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Overwriting /home/yaroslav/.ipython/extensions/my_ext.py\n"
]
}
],
"source": [
"%%file ~/.ipython/extensions/my_ext.py\n",
"\n",
"import re\n",
"import nbformat\n",
"import json\n",
"import subprocess\n",
"from IPython.core.magic import register_cell_magic, register_line_magic\n",
"from IPython.core.display import display, Javascript, HTML\n",
"\n",
"def load_ipython_extension(IPython):\n",
"\n",
" @register_cell_magic('py')\n",
" def css_cell_magic(line, cell):\n",
" IPython.kernel.shell.run_cell(cell, False)\n",
"\n",
" completer = IPython.Completer.splitter\n",
" completer.delims = completer.delims.replace('@', '')\n",
"\n",
" @register_cell_magic('requirejs')\n",
" def requirejs_cell_magic(line, cell):\n",
" global requirejs\n",
" requirejs = {\n",
" 'config': {},\n",
" 'shortcuts': {},\n",
" 'lib_path' : line.split()[0]\n",
" }\n",
" for line in cell.split('\\n'):\n",
" elems = line.split()\n",
" if len(elems) == 2:\n",
" shortcut, file = elems\n",
" requirejs['shortcuts'][shortcut] = file\n",
" elif len(elems) == 3:\n",
" shortcut, import_name, file = elems\n",
" requirejs['shortcuts'][shortcut] = import_name\n",
" if file.endswith('.js'):\n",
" file = file[:-3]\n",
" requirejs['config'][import_name] = file\n",
" if len(requirejs['config']) > 0:\n",
" paths = json.dumps(requirejs['config'], indent=4)\n",
" return Javascript('require.config({paths:\\n %s\\n});' % paths)\n",
"\n",
" def add_requirejs(cell):\n",
" if 'requirejs' not in globals():\n",
" return cell\n",
" shortcuts, files = zip(*requirejs['shortcuts'].items())\n",
" files = ', '.join(['\"%s\"' % file for file in files])\n",
" shortcuts = ', '.join([shortcut for shortcut in shortcuts])\n",
" return 'require([%s], function(%s) {\\n%s\\n})' % (files, shortcuts, cell)\n",
"\n",
" def js_no_out(code):\n",
" code += '; IPython.notebook.events.one(\"output_appended.OutputArea\", function() {'\n",
" code += ' setTimeout(function() {'\n",
" code += ' console.log(\"output_appended\");'\n",
" code += ' var idx = IPython.notebook.get_selected_index() - 1;'\n",
" code += ' IPython.notebook.get_cell(idx).clear_output();'\n",
" code += ' }, 1000)'\n",
" code += '})'\n",
" return code\n",
" \n",
" @register_cell_magic('js')\n",
" def css_cell_magic(line, cell):\n",
" res = add_requirejs(cell)\n",
" if line.startswith('no_out'):\n",
" res = js_no_out(res)\n",
" return Javascript(res)\n",
"\n",
" def prefix_css_file(fname, prefix):\n",
" res = subprocess.run(['/data/sets/venv/node_modules/.bin/prefix-css', prefix, fname], stdout=subprocess.PIPE, stderr=subprocess.PIPE)\n",
" if res.stderr:\n",
" raise Exception(res.stderr.decode('utf8'))\n",
" return res.stdout.decode('utf8')\n",
"\n",
" def prefix_css_source(source, prefix):\n",
" temp_fname = '/data/temp/prefix.css'\n",
" with open(temp_fname, 'w') as f:\n",
" f.write(source)\n",
" return prefix_css_file(temp_fname, prefix)\n",
"\n",
" def split_by_token(line):\n",
" line = line.strip()\n",
" if line == '':\n",
" return ('', '')\n",
" elif line[0] == '\"':\n",
" pos = line.index('\"', 1)\n",
" return (line[1:pos], line[pos+1:])\n",
" elif ' ' in line:\n",
" return line.split(' ', 1)\n",
" else:\n",
" return (line, '')\n",
"\n",
" @register_cell_magic('css')\n",
" def css_cell_magic(line, cell):\n",
" prefix, _ = split_by_token(line)\n",
" if prefix != '' and not prefix.startswith('@'):\n",
" source = prefix_css_source(cell, prefix)\n",
" else:\n",
" source = cell\n",
" try:\n",
" return HTML('<style>\\n' + source + '\\n</style>')\n",
" except Exception as e:\n",
" return HTML('<pre>\\n' + e.args[0] + '\\n</pre>')\n",
"\n",
" @register_line_magic('css')\n",
" def css_line_magic(line):\n",
" fname, rest = split_by_token(line)\n",
" prefix, _ = split_by_token(rest)\n",
" source = open(fname).read()\n",
" if prefix != '' and not prefix.startswith('@'):\n",
" source = prefix_css_source(source, prefix)\n",
" try:\n",
" return HTML('<style>\\n' + source + '\\n</style>')\n",
" except Exception as e:\n",
" return HTML('<pre>\\n' + e.args[0] + '\\n</pre>')\n",
"\n",
" def register_react_worker():\n",
" node_source_location = '/data/temp/node_react_worker.js'\n",
" node_modules_location = '/data/sets/venv/node_modules'\n",
" Bg.inst('react', force_new=True).set_params(\n",
" ['node', node_source_location],\n",
" stdin=subprocess.PIPE,\n",
" stderr=subprocess.PIPE,\n",
" stdout=subprocess.PIPE,\n",
" env = {'NODE_PATH': node_modules_location}\n",
" )\n",
"\n",
" def react_transform(source):\n",
" inst = Bg.inst('react')\n",
" inst.ensure_running()\n",
" msg = json.dumps(source, ensure_ascii=False).encode('utf8') + b'\\n'\n",
" inst.popen.stdin.write(msg)\n",
" inst.popen.stdin.flush()\n",
" res = json.loads(inst.popen.stdout.readline().decode('utf8'))\n",
" if 'result' in res:\n",
" return res['result']\n",
" else:\n",
" raise Exception('Babel error:\\n%s' % res['error'])\n",
"\n",
" register_react_worker()\n",
"\n",
" @register_cell_magic('jsx')\n",
" def react_magic(line, cell):\n",
" try:\n",
" res = react_transform(add_requirejs(cell))\n",
" if line.startswith('no_out'):\n",
" res = js_no_out(res)\n",
" return Javascript(res)\n",
" except Exception as e:\n",
" return HTML('<pre>' + e.args[0] + '</pre>')\n",
"\n",
"class Bg:\n",
" \n",
" prefix = 'bgprocess_'\n",
" \n",
" def inst(name, force_new=False):\n",
" full_name = Bg.prefix + name\n",
" if full_name in globals():\n",
" old_inst = globals()[full_name]\n",
" if not force_new:\n",
" return old_inst\n",
" else:\n",
" old_inst.stop()\n",
" new_inst = Bg(name).set_params(*old_inst.args, **old_inst.kwargs)\n",
" globals()[full_name] = new_inst\n",
" return new_inst\n",
" else:\n",
" new_inst = Bg(name)\n",
" globals()[full_name] = new_inst\n",
" return new_inst\n",
" \n",
" def __init__(self, name):\n",
" self.name = name\n",
" self.popen = None\n",
" self.err_shown = False\n",
" self.args = ()\n",
" self.kwargs = {}\n",
" \n",
" def is_running(self):\n",
" if self.popen is None:\n",
" return False\n",
" ret_code = self.popen.poll()\n",
" if ret_code is None:\n",
" return True\n",
" return False\n",
" \n",
" def errors(self):\n",
" if self.popen is None:\n",
" return None\n",
" ret_code = self.popen.poll()\n",
" if ret_code is None:\n",
" return None\n",
" if self.err_shown:\n",
" return None\n",
" self.err_shown = True\n",
" err = self.popen.communicate()[1]\n",
" if err is not None:\n",
" return \"%s exception (error code %d):\\n%s\" % (self.name, ret_code, err)\n",
" if ret_code < 0:\n",
" return \"%s aborted by signal %d\" % (self.name, ret_code)\n",
" \n",
" def stop(self):\n",
" if self.is_running():\n",
" self.popen.kill()\n",
" \n",
" def set_params(self, *args, **kwargs):\n",
" self.args = args\n",
" self.kwargs = kwargs\n",
" return self\n",
" \n",
" def run(self, *args, **kwargs):\n",
" if self.is_running():\n",
" return\n",
" self.popen = subprocess.Popen(*self.args, **self.kwargs)\n",
" self.err_shown = False\n",
" \n",
" def ensure_running(self):\n",
" err = self.errors()\n",
" if err is not None:\n",
" raise Exception(err)\n",
" if not self.is_running():\n",
" print('Starting %s...' % self.name)\n",
" self.run()\n",
"\n",
"class NotebookCell:\n",
" \n",
" def __init__(self, cell, parents, index):\n",
" self.index = index\n",
" self.cell = cell\n",
" self.labels = set()\n",
" self.magic = None\n",
" self.kind = cell['cell_type']\n",
" if self.kind == 'markdown' and '\\n' not in cell['source'] and cell['source'].startswith('#'):\n",
" self.kind = 'title'\n",
" self.level = cell['source'].count('#')\n",
" items = list(parents.items())\n",
" for level, _ in items:\n",
" if level >= self.level:\n",
" del parents[level]\n",
" parents[self.level] = self\n",
" self.labels = set(re.findall('@(\\w+)', cell['source']))\n",
" self.source = cell['source'].replace('#', '').split('@')[0].strip()\n",
" self.parents = [parents[level] for level in sorted(parents.keys())]\n",
" if self.kind not in ['code', 'raw']:\n",
" return\n",
" if cell['source'].startswith('%%') and '\\n' in cell['source']:\n",
" self.top_line, self.source = cell['source'].split('\\n', 1)\n",
" self.magic = self.top_line.split()[0]\n",
" self.labels = set(re.findall('@\\w+', self.top_line))\n",
" else:\n",
" self.source = cell['source']\n",
"\n",
" def last_label(self, *args):\n",
" res = args[0]\n",
" for parent in self.parents:\n",
" inter = parent.labels.intersect(labels)\n",
" if len(inter) > 1:\n",
" raise Exception('More than one label: %s', str(inter))\n",
" elif len(inter) == 1:\n",
" res = inter.pop()\n",
" return res\n",
" \n",
" def index_str(self):\n",
" return '#%d [%s]' % (self.index, str(self.cell['execution_count']))\n",
" \n",
" def snippet(self):\n",
" return self.cell['source'].replace('\\n', '; ')[:50]\n",
" \n",
" def get_output(self, mime):\n",
" try:\n",
" for output in self.cell['outputs']:\n",
" if 'data' in output:\n",
" if mime in output['data']:\n",
" return output['data'][mime]\n",
" raise Exception()\n",
" except:\n",
" raise Exception('Can\\'t get output of cell %s:\\n%s' % (self.index_str(), self.snippet()))\n"
]
},
{
"cell_type": "code",
"execution_count": 40,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Writing /data/temp/node_react_worker.js\n"
]
}
],
"source": [
"%%file /data/temp/node_react_worker.js\n",
"\n",
"var readline = require('readline');\n",
"var babel = require(\"babel-core\");\n",
"\n",
"var rl = readline.createInterface({\n",
" input: process.stdin,\n",
" output: process.stdout,\n",
" terminal: false\n",
"});\n",
"\n",
"rl.on('line', function(line) {\n",
" try {\n",
" var res = babel.transform(JSON.parse(line), {\n",
" presets: ['es2015', 'react']\n",
" }).code;\n",
" process.stdout.write(JSON.stringify({result: res}) + '\\n');\n",
" } catch (e) {\n",
" process.stdout.write(JSON.stringify({error: e.stack}) + '\\n');\n",
" }\n",
"})"
]
},
{
"cell_type": "code",
"execution_count": 87,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Overwriting /home/yaroslav/.jupyter/custom/patch.css\n"
]
}
],
"source": [
"%%file ~/.jupyter/custom/patch.css\n",
".code_cell.init-cell .inner_cell .CodeMirror-gutters {\n",
"/* border-right: 1px solid lightgray; */\n",
" background: #544;\n",
"}\n",
".CodeMirror-gutter-wrapper {\n",
" transform: translate(-4px, 0px);\n",
"}\n",
".code_cell .inner_cell .CodeMirror-gutters {\n",
"/* border-right: 1px solid black; */\n",
"}\n",
".CodeMirror span.cm-qualifier {\n",
" color: lightgray;\n",
"}\n",
".CodeMirror span.cm-comment {\n",
" color: gray;\n",
"}\n",
"div#pager .ui-resizable-handle {\n",
" background: #272727;\n",
" border-color: #777;\n",
"}\n",
"div#pager .ui-resizable-handle::after {\n",
" border-top: 1px solid #777;\n",
"}\n",
"#pager-contents {\n",
" background: #111;\n",
"}\n",
"div#pager-container pre {\n",
" background: #272727;\n",
" box-shadow: 0px 0px 12px 1px rgba(87, 87, 87, 0.2);\n",
" border: none;\n",
" color: #F8F8F8;\n",
"}\n",
"#tooltip {\n",
" background: #272727;\n",
" border: 1px solid #777;\n",
" color: #F8F8F8;\n",
" box-shadow: 0px 0px 12px 1px rgba(87, 87, 87, 0.2);\n",
"}\n",
"#tooltip .tooltiparrow{\n",
" color: #F8F8F8;\n",
"}\n",
"\n",
"#tooltip .pretooltiparrow:before {\n",
" background-color : #272727;\n",
" border : 1px solid #777;\n",
"}\n",
"\n",
"#tooltip .tooltiptext pre {\n",
" background-color : #272727;\n",
" border : none;\n",
" color: #F8F8F8;\n",
"}\n",
".fileinput {\n",
" height: 18px\n",
"}"
]
},
{
"cell_type": "code",
"execution_count": 88,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"! cd ~/.jupyter/custom && cat dark_theme.css patch.css > custom.css"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "py3",
"language": "python",
"name": "py3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.5.2"
}
},
"nbformat": 4,
"nbformat_minor": 1
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment