shared memory fun in python

readme.txt

using posix_ipc get some shared memory. ended up using fcntl.lockf to handle locking as it gives us locking on
ranges, which really helps performance. granted, performance isn't a big concern for this project, but it was
rather annoying how quickly things degraded when using other methods.

reader.py

import ntxop.smem as smem

TACOS = (0, 32, smem._STR)
FOO = (32, 48, smem._INT)
BAR = (48, 65584, smem._PKL)
QUUX = (65584, 65600, smem._STR)

mm = smem.SmemFile('/mangos', 'r')


_zz = { 'foo' : { 'bar' : 33, 'baz' : 'quux' }, 'dink' : 'donk' }

c = 0
while (c<1000000):
        print(mm.get_offset(TACOS))
        c+=1

setter.py

import ntxop.smem as smem

TACOS = (0, 32, smem._STR)
FOO = (32, 48, smem._INT)
BAR = (48, 65584, smem._PKL)
QUUX = (65584, 65600, smem._STR)

mm = smem.SmemFile('/mangos', 'w')


_zz = { 'foo' : { 'bar' : 33, 'baz' : 'quux' }, 'dink' : 'donk' }

c = 0
while (c<1000000):
        for x in ["A"*32, "B"*32, 'C'*32, 'D'*32, 'E'*32]:
                mm.set_offset(TACOS,x)
        c+=1

setter2.py

import ntxop.smem as smem

TACOS = (0, 32, smem._STR)
FOO = (32, 48, smem._INT)
BAR = (48, 65584, smem._PKL)
QUUX = (65584, 65800, smem._STR)

mm = smem.SmemFile('/mangos', 'w')


_zz = { 'foo' : { 'bar' : 33, 'baz' : 'quux' }, 'dink' : 'donk' }

c = 0
while (c<1000000):
        for x in ["A"*32, "B"*32, 'C'*32, 'D'*32, 'E'*32]:
                mm.set_offset(QUUX,x)
        c+=1

smem.py

"""
The smem module provides a standard interface to a shared
memory buffer.  Any change to offsets or resizing the 
buffer will require a cold start of the system
"""

import mmap
import pickle
import fcntl
import posix_ipc

_INT = 0 # int-as-string
_STR = 1 # string
_PKL = 2 # pickled
_JSN = 3 # json is unimplemenetd
_FLT = 4 # floating point
_RAW = 5 # just return it...

_SHMSIZE = 32768*1024 # 32MB!

# GENERATION = (0,8, _STR)
# LENGTH = (8,16, _INT)
class SmemFile:
    """
    SmemFile deals with the actual file and is generic enough to be used
    with anything that needs to deal with regions in an mmap file in the
    simple ways we use them in ntxop
    """    

    def __init__(self, shmname, mode="r"):
        """
        Opens the given file then mmaps it to a buffer in the object
        mode can be 'w' for read/write or anything else for read-only.
        The default is read-only.
        """
        if ( shmname[0] == "/" ):
             self._shmname = shmname
             self._semname = shmname # it gets named sem.whatever so nbd
        else:
             raise ValueError('Invalid shm name')

        if (mode == 'w'):
            prot = mmap.PROT_READ | mmap.PROT_WRITE
            self._shm = posix_ipc.SharedMemory(self._shmname, posix_ipc.O_CREAT | posix_ipc.O_RDWR, 0o664, _SHMSIZE)
        else:
            prot = mmap.PROT_READ
            self._shm = posix_ipc.SharedMemory(self._shmname, posix_ipc.O_CREAT | posix_ipc.O_RDWR, 0o664, _SHMSIZE, True)
            
        self.buf = mmap.mmap(self._shm.fd, 0, prot=prot)
        
    def flush(self):
        self.buf.flush()

    def get_offset(self, offset):
        """
        return the item at the given offset. The offset is a tuple
        in the form of (start, end, type).  Types are defined
        at the top of this file: _INT, _STR, _PKL, _JSN, _FLT

        strings are always returned stripped. do your formatting
        elsewhere or do something more clever.
        """
        start, end, typ = offset

        maxlen = end - start

        fcntl.lockf(self._shm.fd, fcntl.LOCK_SH, maxlen, start, 0)
        buf = self.buf[start:end]
        fcntl.lockf(self._shm.fd, fcntl.LOCK_UN, maxlen, start, 0)
        
        # strings are stripped and decoded
        if (typ == _STR):
            return buf.strip(b'\x00 ').decode()

        # invalid number strings get returned as zero
        if (typ == _INT):
            try:
                return int(buf.strip(b'\x00 '))
            except:
                return 0

        # invalid pickles return as None
        # as the assumption is that nothing is there
        if (typ == _PKL):
            try:
                return pickle.loads(buf)
            except:
                return None

        # return 0.0 on invalid floating point
        if (typ == _FLT):
            try:
                return float(buf.strip(b'\x00 '))
            except:
                return float(0)

        # assume raw by default
        return buf

    def set_offset(self, offset, obj):
        """
        the setting version of get_offset... the obj is
        handled based on the type specified in the offset tuple
        """
        
        start, end, typ = offset

        # we can't put more stuff into the shared buffer than
        # is allowed so let's get our max length here
        maxlen = end - start

        # fcntl.lockf(fd, cmd, len=0, start=0, whence=0)

 
        if (typ == _STR):
            if (len(obj) > maxlen):
                raise ValueError('TOO LONG')
            fcntl.lockf(self._shm.fd, fcntl.LOCK_EX, maxlen, start, 0)
            self.buf[start:end] = bytes(str(obj), 'utf-8').ljust(maxlen, b'\x00')
            fcntl.lockf(self._shm.fd, fcntl.LOCK_UN, maxlen, start, 0)
            return True
        if (typ == _INT):
            tmp = str(obj)
            if (len(tmp) > maxlen):
                raise ValueError('TOO LONG')
            fcntl.lockf(self._shm.fd, fcntl.LOCK_EX, maxlen, start, 0)
            self.buf[start:end] = bytes(tmp, 'utf-8').ljust(maxlen, b'\x00')
            fcntl.lockf(self._shm.fd, fcntl.LOCK_UN, maxlen, start, 0)
            return True
        if (typ == _PKL):
            # handles like raw but with an extra step...
            tmp = pickle.dumps(obj)
            if (len(tmp) > maxlen):
                raise ValueError('TOO LONG')
            self._write(start, maxlen, tmp)
            return True
        if (typ == _JSN):
            return False
        if (typ == _FLT):
            return False
        if (typ == _RAW):
            if (len(obj) > maxlen):
                raise ValueError('TOO LONG')
            self._write(start, maxlen, obj)
            return True
        raise ValueError("Invalid type sepcified")


    def _write(self, start, blen, blob):
        """
        does a write to the buffer like a file
        """
        if ( len(blob) > blen ):
            raise ValueError
        fcntl.lockf(self._shm.fd, fcntl.LOCK_EX, maxlen, start, 0)
        self.buf[start:start+blen] = b''.ljust(blen, b'\x00')
        self.buf.seek(start)
        self.buf.write(blob)
        fcntl.lockf(self._shm.fd, fcntl.LOCK_UN, maxlen, start, 0)
        return self.buf.seek(0)