persistent bash magics for jupyter notebook.

bash_kernel_wrapper.py

# Based on https://github.com/takluyver/bash_kernel/blob/master/bash_kernel/kernel.py
# But whereas bash_kernel interits BashKernel(Kernel), we must inject the kernel in the constructor
# BashKernel(object) and __init__(self, kernel, **kwargs)

from IPython.kernel.zmq.kernelbase import Kernel
from pexpect import replwrap, EOF

from subprocess import check_output
from os import unlink

import base64
import imghdr
import re
import signal
import urllib

__version__ = '0.2'

version_pat = re.compile(r'version (\d+(\.\d+)+)')

from bash_kernel.images import (
    extract_image_filenames, display_data_for_image, image_setup_cmd
)


class BashKernel(object):
    implementation = 'bash_kernel'
    implementation_version = __version__

    @property
    def language_version(self):
        m = version_pat.search(self.banner)
        return m.group(1)

    _banner = None

    @property
    def banner(self):
        if self._banner is None:
            self._banner = check_output(['bash', '--version']).decode('utf-8')
        return self._banner

    language_info = {'name': 'bash',
                     'codemirror_mode': 'shell',
                     'mimetype': 'text/x-sh',
                     'file_extension': '.sh'}

    def __init__(self, kernel,**kwargs):
        #Kernel.__init__(self, **kwargs)
        self.kernel = kernel
        self._start_bash()

    def _start_bash(self):
        # Signal handlers are inherited by forked processes, and we can't easily
        # reset it from the subprocess. Since kernelapp ignores SIGINT except in
        # message handlers, we need to temporarily reset the SIGINT handler here
        # so that bash and its children are interruptible.
        sig = signal.signal(signal.SIGINT, signal.SIG_DFL)
        try:
            self.bashwrapper = replwrap.bash()
        finally:
            signal.signal(signal.SIGINT, sig)

        # Register Bash function to write image data to temporary file
        self.bashwrapper.run_command(image_setup_cmd)

    def do_execute(self, code, silent, store_history=True,
                   user_expressions=None, allow_stdin=False):
        if not code.strip():
            return {'status': 'ok', 'execution_count': self.kernel.execution_count,
                    'payload': [], 'user_expressions': {}}

        interrupted = False
        try:
            output = self.bashwrapper.run_command(code.rstrip(), timeout=None)
        except KeyboardInterrupt:
            self.bashwrapper.child.sendintr()
            interrupted = True
            self.bashwrapper._expect_prompt()
            output = self.kernel.bashwrapper.child.before
        except EOF:
            output = self.bashwrapper.child.before + 'Restarting Bash'
            self._start_bash()

        if not silent:
            image_filenames, output = extract_image_filenames(output)

            # Send standard output
            stream_content = {'name': 'stdout', 'text': output}
            self.kernel.send_response(self.kernel.iopub_socket, 'stream', stream_content)

            # Send images, if any
            for filename in image_filenames:
                try:
                    data = display_data_for_image(filename)
                except ValueError as e:
                    message = {'name': 'stdout', 'text': str(e)}
                    self.kernel.send_response(self.kernel.iopub_socket, 'stream', message)
                else:
                    self.kernel.send_response(self.kernel.iopub_socket, 'display_data', data)

        if interrupted:
            return {'status': 'abort', 'execution_count': self.kernel.execution_count}

        try:
            exitcode = int(self.bashwrapper.run_command('echo $?').rstrip())
        except Exception:
            exitcode = 1

        if exitcode:
            return {'status': 'error', 'execution_count': self.kernel.execution_count,
                    'ename': '', 'evalue': str(exitcode), 'traceback': []}
        else:
            return {'status': 'ok', 'execution_count': self.kernel.execution_count,
                    'payload': [], 'user_expressions': {}}

    def do_complete(self, code, cursor_pos):
        code = code[:cursor_pos]
        default = {'matches': [], 'cursor_start': 0,
                   'cursor_end': cursor_pos, 'metadata': dict(),
                   'status': 'ok'}

        if not code or code[-1] == ' ':
            return default

        tokens = code.replace(';', ' ').split()
        if not tokens:
            return default

        matches = []
        token = tokens[-1]
        start = cursor_pos - len(token)

        if token[0] == '$':
            # complete variables
            cmd = 'compgen -A arrayvar -A export -A variable %s' % token[1:] # strip leading $
            output = self.bashwrapper.run_command(cmd).rstrip()
            completions = set(output.split())
            # append matches including leading $
            matches.extend(['$'+c for c in completions])
        else:
            # complete functions and builtins
            cmd = 'compgen -cdfa %s' % token
            output = self.bashwrapper.run_command(cmd).rstrip()
            matches.extend(output.split())
            
        if not matches:
            return default
        matches = [m for m in matches if m.startswith(token)]

        return {'matches': sorted(matches), 'cursor_start': start,
                'cursor_end': cursor_pos, 'metadata': dict(),
                'status': 'ok'}

discussion.py

 """
 After installing bash_kernel https://github.com/takluyver/bash_kernel we can modify the kernel.py to have a 
 "has a" kernel rather than "is a" kernel relationship and things will work.
 
 Naively, I'd hoped to be able to just edit kernel.py as follows:
     def __init__(self, **kwargs):
        - Kernel.__init__(self, **kwargs)
        + if not Kernel.initialized():
        +    Kernel.__init__(self, **kwargs)
        self._start_bash()
 
However, the test here triggers the error below 
######################## RUN TEST CELL ################## 
%%pbash
export TEST=val
echo $TEST
#########################################################
---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-7-8aa16948a72f> in <module>()
----> 1 get_ipython().run_cell_magic('pbcmagic', '', 'export TEST=val\necho $TEST')

/Users/kestenbroughton/anaconda3/lib/python3.4/site-packages/IPython/core/interactiveshell.py in run_cell_magic(self, magic_name, line, cell)
   2262             magic_arg_s = self.var_expand(line, stack_depth)
   2263             with self.builtin_trap:
-> 2264                 result = fn(magic_arg_s, cell)
   2265             return result
   2266 

<string> in pbcmagic(self, line, cell)

/Users/kestenbroughton/anaconda3/lib/python3.4/site-packages/IPython/core/magic.py in <lambda>(f, *a, **k)
    191     # but it's overkill for just that one bit of state.
    192     def magic_deco(arg):
--> 193         call = lambda f, *a, **k: f(*a, **k)
    194 
    195         if callable(arg):

<ipython-input-6-0ebe4aefcf32> in pbcmagic(self, line, cell)
     22     @cell_magic
     23     def pbcmagic(self, line, cell):
---> 24         self.bash_kernel.do_execute(cell, False)
     25         return line, cell
     26 

<ipython-input-3-513115b5dd5f> in do_execute(self, code, silent, store_history, user_expressions, allow_stdin)
     85             # Send standard output
     86             stream_content = {'name': 'stdout', 'text': output}
---> 87             self.send_response(self.iopub_socket, 'stream', stream_content)
     88 
     89             # Send images, if any

/Users/kestenbroughton/anaconda3/lib/python3.4/site-packages/IPython/kernel/zmq/kernelbase.py in send_response(self, stream, msg_or_type, content, ident, buffers, track, header, metadata)
    331         message.
    332         """
--> 333         return self.session.send(stream, msg_or_type, content, self._parent_header,
    334                                  ident, buffers, track, header, metadata)
    335 

AttributeError: 'NoneType' object has no attribute 'send'       
 """
 
 # This is caused because     
 # session = Instance(Session)
 # returns None, likely because session is a singleton?
 # /Users/kestenbroughton/anaconda3/lib/python3.4/site-packages/IPython/kernel/zmq/kernelbase.py
 
 class Kernel(SingletonConfigurable):

    #---------------------------------------------------------------------------
    # Kernel interface
    #---------------------------------------------------------------------------

    # attribute to override with a GUI
    eventloop = Any(None)
    def _eventloop_changed(self, name, old, new):
        """schedule call to eventloop from IOLoop"""
        loop = ioloop.IOLoop.instance()
        loop.add_callback(self.enter_eventloop)

    session = Instance(Session)
    profile_dir = Instance('IPython.core.profiledir.ProfileDir')
    shell_streams = List()

questions.txt

I was following https://ipython.org/ipython-doc/dev/config/custommagics.html

1. Docs should note that for ClassMagics you do not delete the names as you do for function magics
# We delete these to avoid name conflicts for automagic to work
del lmagic, lcmagic

2.  Is there a way to supress the output showing the (line,cell) inputs?
('', 'export TEST=val\necho $TEST') 

3. Is the current bash magics non-persistent by design, or would a PR be welcome for this?

4. The bash_kernel relies heaviliy on kernel functionality. Is there a way to cleanly factor the code to avoid
   "has a" semantics for magics but "is a" semantics for using bash as a primary kernel?

stateful_bash_magics.py

from __future__ import print_function
from IPython.core.magic import (Magics, magics_class, line_magic,
                                cell_magic, line_cell_magic)

@magics_class
class StatefulBashMagics(Magics):
    "Magics that hold additional state"

    # Singleton bash kernel for all cell and line magics in notebook
    bash_kernel = BashKernel(get_ipython().kernel)
    
    def __init__(self, shell, data):
        # You must call the parent constructor
        super(StatefulBashMagics, self).__init__(shell)
        self.data = data
        
    
    @line_magic
    def plbash(self, line):
        self.bash_kernel.do_execute(line, False)
        return line

    @cell_magic
    def pcbash(self, line, cell):
        self.bash_kernel.do_execute(cell, False)
        return line, cell

    @line_cell_magic
    def pbash(self, line, cell=None):
        "Magic that works both as %pblmagic and as %%pbccmagic"
        if cell is None:
            self.bash_kernel.do_execute(line, False)
            return line
        else:
            self.bash_kernel.do_execute(line, cell, False)
            return line, cell

ip = get_ipython()
magics = StatefulBashMagics(ip)
ip.register_magics(magics)

test.sh

# Successful test shows TEST defined in one cell is still defined in next cell.

%%pbash
export TEST=val
echo $TEST
val
Out[3]: ('', 'export TEST=val\necho $TEST')


%pbash echo $TEST
val
Out[4]: 'echo $TEST'