Last active
June 4, 2022 00:48
-
-
Save kbroughton/e82a9e381cdf228eb380 to your computer and use it in GitHub Desktop.
persistent bash magics for jupyter notebook.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Based on https://github.com/takluyver/bash_kernel/blob/master/bash_kernel/kernel.py | |
# But whereas bash_kernel interits BashKernel(Kernel), we must inject the kernel in the constructor | |
# BashKernel(object) and __init__(self, kernel, **kwargs) | |
from IPython.kernel.zmq.kernelbase import Kernel | |
from pexpect import replwrap, EOF | |
from subprocess import check_output | |
from os import unlink | |
import base64 | |
import imghdr | |
import re | |
import signal | |
import urllib | |
__version__ = '0.2' | |
version_pat = re.compile(r'version (\d+(\.\d+)+)') | |
from bash_kernel.images import ( | |
extract_image_filenames, display_data_for_image, image_setup_cmd | |
) | |
class BashKernel(object): | |
implementation = 'bash_kernel' | |
implementation_version = __version__ | |
@property | |
def language_version(self): | |
m = version_pat.search(self.banner) | |
return m.group(1) | |
_banner = None | |
@property | |
def banner(self): | |
if self._banner is None: | |
self._banner = check_output(['bash', '--version']).decode('utf-8') | |
return self._banner | |
language_info = {'name': 'bash', | |
'codemirror_mode': 'shell', | |
'mimetype': 'text/x-sh', | |
'file_extension': '.sh'} | |
def __init__(self, kernel,**kwargs): | |
#Kernel.__init__(self, **kwargs) | |
self.kernel = kernel | |
self._start_bash() | |
def _start_bash(self): | |
# Signal handlers are inherited by forked processes, and we can't easily | |
# reset it from the subprocess. Since kernelapp ignores SIGINT except in | |
# message handlers, we need to temporarily reset the SIGINT handler here | |
# so that bash and its children are interruptible. | |
sig = signal.signal(signal.SIGINT, signal.SIG_DFL) | |
try: | |
self.bashwrapper = replwrap.bash() | |
finally: | |
signal.signal(signal.SIGINT, sig) | |
# Register Bash function to write image data to temporary file | |
self.bashwrapper.run_command(image_setup_cmd) | |
def do_execute(self, code, silent, store_history=True, | |
user_expressions=None, allow_stdin=False): | |
if not code.strip(): | |
return {'status': 'ok', 'execution_count': self.kernel.execution_count, | |
'payload': [], 'user_expressions': {}} | |
interrupted = False | |
try: | |
output = self.bashwrapper.run_command(code.rstrip(), timeout=None) | |
except KeyboardInterrupt: | |
self.bashwrapper.child.sendintr() | |
interrupted = True | |
self.bashwrapper._expect_prompt() | |
output = self.kernel.bashwrapper.child.before | |
except EOF: | |
output = self.bashwrapper.child.before + 'Restarting Bash' | |
self._start_bash() | |
if not silent: | |
image_filenames, output = extract_image_filenames(output) | |
# Send standard output | |
stream_content = {'name': 'stdout', 'text': output} | |
self.kernel.send_response(self.kernel.iopub_socket, 'stream', stream_content) | |
# Send images, if any | |
for filename in image_filenames: | |
try: | |
data = display_data_for_image(filename) | |
except ValueError as e: | |
message = {'name': 'stdout', 'text': str(e)} | |
self.kernel.send_response(self.kernel.iopub_socket, 'stream', message) | |
else: | |
self.kernel.send_response(self.kernel.iopub_socket, 'display_data', data) | |
if interrupted: | |
return {'status': 'abort', 'execution_count': self.kernel.execution_count} | |
try: | |
exitcode = int(self.bashwrapper.run_command('echo $?').rstrip()) | |
except Exception: | |
exitcode = 1 | |
if exitcode: | |
return {'status': 'error', 'execution_count': self.kernel.execution_count, | |
'ename': '', 'evalue': str(exitcode), 'traceback': []} | |
else: | |
return {'status': 'ok', 'execution_count': self.kernel.execution_count, | |
'payload': [], 'user_expressions': {}} | |
def do_complete(self, code, cursor_pos): | |
code = code[:cursor_pos] | |
default = {'matches': [], 'cursor_start': 0, | |
'cursor_end': cursor_pos, 'metadata': dict(), | |
'status': 'ok'} | |
if not code or code[-1] == ' ': | |
return default | |
tokens = code.replace(';', ' ').split() | |
if not tokens: | |
return default | |
matches = [] | |
token = tokens[-1] | |
start = cursor_pos - len(token) | |
if token[0] == '$': | |
# complete variables | |
cmd = 'compgen -A arrayvar -A export -A variable %s' % token[1:] # strip leading $ | |
output = self.bashwrapper.run_command(cmd).rstrip() | |
completions = set(output.split()) | |
# append matches including leading $ | |
matches.extend(['$'+c for c in completions]) | |
else: | |
# complete functions and builtins | |
cmd = 'compgen -cdfa %s' % token | |
output = self.bashwrapper.run_command(cmd).rstrip() | |
matches.extend(output.split()) | |
if not matches: | |
return default | |
matches = [m for m in matches if m.startswith(token)] | |
return {'matches': sorted(matches), 'cursor_start': start, | |
'cursor_end': cursor_pos, 'metadata': dict(), | |
'status': 'ok'} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
""" | |
After installing bash_kernel https://github.com/takluyver/bash_kernel we can modify the kernel.py to have a | |
"has a" kernel rather than "is a" kernel relationship and things will work. | |
Naively, I'd hoped to be able to just edit kernel.py as follows: | |
def __init__(self, **kwargs): | |
- Kernel.__init__(self, **kwargs) | |
+ if not Kernel.initialized(): | |
+ Kernel.__init__(self, **kwargs) | |
self._start_bash() | |
However, the test here triggers the error below | |
######################## RUN TEST CELL ################## | |
%%pbash | |
export TEST=val | |
echo $TEST | |
######################################################### | |
--------------------------------------------------------------------------- | |
AttributeError Traceback (most recent call last) | |
<ipython-input-7-8aa16948a72f> in <module>() | |
----> 1 get_ipython().run_cell_magic('pbcmagic', '', 'export TEST=val\necho $TEST') | |
/Users/kestenbroughton/anaconda3/lib/python3.4/site-packages/IPython/core/interactiveshell.py in run_cell_magic(self, magic_name, line, cell) | |
2262 magic_arg_s = self.var_expand(line, stack_depth) | |
2263 with self.builtin_trap: | |
-> 2264 result = fn(magic_arg_s, cell) | |
2265 return result | |
2266 | |
<string> in pbcmagic(self, line, cell) | |
/Users/kestenbroughton/anaconda3/lib/python3.4/site-packages/IPython/core/magic.py in <lambda>(f, *a, **k) | |
191 # but it's overkill for just that one bit of state. | |
192 def magic_deco(arg): | |
--> 193 call = lambda f, *a, **k: f(*a, **k) | |
194 | |
195 if callable(arg): | |
<ipython-input-6-0ebe4aefcf32> in pbcmagic(self, line, cell) | |
22 @cell_magic | |
23 def pbcmagic(self, line, cell): | |
---> 24 self.bash_kernel.do_execute(cell, False) | |
25 return line, cell | |
26 | |
<ipython-input-3-513115b5dd5f> in do_execute(self, code, silent, store_history, user_expressions, allow_stdin) | |
85 # Send standard output | |
86 stream_content = {'name': 'stdout', 'text': output} | |
---> 87 self.send_response(self.iopub_socket, 'stream', stream_content) | |
88 | |
89 # Send images, if any | |
/Users/kestenbroughton/anaconda3/lib/python3.4/site-packages/IPython/kernel/zmq/kernelbase.py in send_response(self, stream, msg_or_type, content, ident, buffers, track, header, metadata) | |
331 message. | |
332 """ | |
--> 333 return self.session.send(stream, msg_or_type, content, self._parent_header, | |
334 ident, buffers, track, header, metadata) | |
335 | |
AttributeError: 'NoneType' object has no attribute 'send' | |
""" | |
# This is caused because | |
# session = Instance(Session) | |
# returns None, likely because session is a singleton? | |
# /Users/kestenbroughton/anaconda3/lib/python3.4/site-packages/IPython/kernel/zmq/kernelbase.py | |
class Kernel(SingletonConfigurable): | |
#--------------------------------------------------------------------------- | |
# Kernel interface | |
#--------------------------------------------------------------------------- | |
# attribute to override with a GUI | |
eventloop = Any(None) | |
def _eventloop_changed(self, name, old, new): | |
"""schedule call to eventloop from IOLoop""" | |
loop = ioloop.IOLoop.instance() | |
loop.add_callback(self.enter_eventloop) | |
session = Instance(Session) | |
profile_dir = Instance('IPython.core.profiledir.ProfileDir') | |
shell_streams = List() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
I was following https://ipython.org/ipython-doc/dev/config/custommagics.html | |
1. Docs should note that for ClassMagics you do not delete the names as you do for function magics | |
# We delete these to avoid name conflicts for automagic to work | |
del lmagic, lcmagic | |
2. Is there a way to supress the output showing the (line,cell) inputs? | |
('', 'export TEST=val\necho $TEST') | |
3. Is the current bash magics non-persistent by design, or would a PR be welcome for this? | |
4. The bash_kernel relies heaviliy on kernel functionality. Is there a way to cleanly factor the code to avoid | |
"has a" semantics for magics but "is a" semantics for using bash as a primary kernel? |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from __future__ import print_function | |
from IPython.core.magic import (Magics, magics_class, line_magic, | |
cell_magic, line_cell_magic) | |
@magics_class | |
class StatefulBashMagics(Magics): | |
"Magics that hold additional state" | |
# Singleton bash kernel for all cell and line magics in notebook | |
bash_kernel = BashKernel(get_ipython().kernel) | |
def __init__(self, shell, data): | |
# You must call the parent constructor | |
super(StatefulBashMagics, self).__init__(shell) | |
self.data = data | |
@line_magic | |
def plbash(self, line): | |
self.bash_kernel.do_execute(line, False) | |
return line | |
@cell_magic | |
def pcbash(self, line, cell): | |
self.bash_kernel.do_execute(cell, False) | |
return line, cell | |
@line_cell_magic | |
def pbash(self, line, cell=None): | |
"Magic that works both as %pblmagic and as %%pbccmagic" | |
if cell is None: | |
self.bash_kernel.do_execute(line, False) | |
return line | |
else: | |
self.bash_kernel.do_execute(line, cell, False) | |
return line, cell | |
ip = get_ipython() | |
magics = StatefulBashMagics(ip) | |
ip.register_magics(magics) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Successful test shows TEST defined in one cell is still defined in next cell. | |
%%pbash | |
export TEST=val | |
echo $TEST | |
val | |
Out[3]: ('', 'export TEST=val\necho $TEST') | |
%pbash echo $TEST | |
val | |
Out[4]: 'echo $TEST' |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment