Obnam (http://liw.fi/obnam/) plugin to mount a backup generation as FUSE filesystem
| # Copyright (C) 2013 Valery Yundin | |
| # | |
| # This program is free software: you can redistribute it and/or modify | |
| # it under the terms of the GNU General Public License as published by | |
| # the Free Software Foundation, either version 3 of the License, or | |
| # (at your option) any later version. | |
| # | |
| # This program is distributed in the hope that it will be useful, | |
| # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
| # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
| # GNU General Public License for more details. | |
| # | |
| # You should have received a copy of the GNU General Public License | |
| # along with this program. If not, see <http://www.gnu.org/licenses/>. | |
| import os | |
| import stat | |
| import sys | |
| import logging | |
| import errno | |
| import struct | |
| import obnamlib | |
| try: | |
| import fuse | |
| fuse.fuse_python_api = (0, 2) | |
| except ImportError: | |
| class Bunch: | |
| def __init__(self, **kwds): | |
| self.__dict__.update(kwds) | |
| fuse = Bunch(Fuse = object) | |
| class ObnamFuseOptParse(object): | |
| '''Option parsing class for FUSE | |
| has to set fuse_args.mountpoint | |
| ''' | |
| obnam = None | |
| def __init__(self, *args, **kw): | |
| self.fuse_args = \ | |
| 'fuse_args' in kw and kw.pop('fuse_args') or fuse.FuseArgs() | |
| if 'fuse' in kw: | |
| self.fuse = kw.pop('fuse') | |
| def parse_args(self, args=None, values=None): | |
| self.fuse_args.mountpoint = self.obnam.app.settings['to'] | |
| for opt in self.obnam.app.settings['fuse-opt']: | |
| if opt == '-f': | |
| self.fuse_args.setmod('foreground') | |
| else: | |
| self.fuse_args.add(opt) | |
| if not hasattr(self.fuse_args, 'ro'): | |
| self.fuse_args.add('ro') | |
| class ObnamFuseFile(object): | |
| fs = None # points to active ObnamFuse object | |
| direct_io = False # do not use direct I/O on this file. | |
| keep_cache = True # cached file data need not to be invalidated. | |
| def __init__(self, path, flags, *mode): | |
| logging.debug('FUSE file open %s %d', path, flags) | |
| if ((flags & os.O_WRONLY) or (flags & os.O_RDWR) or | |
| (flags & os.O_CREAT) or (flags & os.O_EXCL) or | |
| (flags & os.O_TRUNC) or (flags & os.O_APPEND)): | |
| raise IOError(errno.EROFS, 'Read only filesystem') | |
| try: | |
| self.path = path | |
| if path == '/.pid' and self.fs.obnam.app.settings['viewmode'] == 'multiple': | |
| self.read = self.read_pid | |
| self.release = self.release_pid | |
| return | |
| self.metadata = self.fs.get_metadata(path) | |
| # if not a regular file return EINVAL | |
| if not stat.S_ISREG(self.metadata.st_mode): | |
| raise IOError(errno.EINVAL, 'Invalid argument') | |
| self.chunkids = None | |
| self.chunksize = None | |
| self.lastdata = None | |
| self.lastblock = None | |
| except: | |
| logging.error('Unexpected exception', exc_info=True) | |
| raise | |
| def read_pid(self, length, offset): | |
| logging.debug('FUSE read_pid %d %d', length, offset) | |
| pid = str(os.getpid()) | |
| if length < len(pid) or offset != 0: | |
| return '' | |
| else: | |
| return pid | |
| def release_pid(self, flags): | |
| self.fs.root_refresh() | |
| return 0 | |
| def fgetattr(self): | |
| logging.debug('FUSE file fgetattr') | |
| return self.fs.getattr(self.path) | |
| def read(self, length, offset): | |
| logging.debug('FUSE file read(%s, %d, %d)', self.path, length, offset) | |
| try: | |
| if length == 0 or offset >= self.metadata.st_size: | |
| return '' | |
| repo = self.fs.obnam.repo | |
| gen, repopath = self.fs.get_gen_path(self.path) | |
| # if stored inside B-tree | |
| contents = repo.get_file_data(gen, repopath) | |
| if contents is not None: | |
| return contents[offset:offset+length] | |
| # stored in chunks | |
| if not self.chunkids: | |
| self.chunkids = repo.get_file_chunks(gen, repopath) | |
| if len(self.chunkids) == 1: | |
| if not self.lastdata: | |
| self.lastdata = repo.get_chunk(self.chunkids[0]) | |
| return self.lastdata[offset:offset+length] | |
| else: | |
| chunkdata = None | |
| if not self.chunksize: | |
| # take the cached value as the first guess for chunksize | |
| self.chunksize = self.fs.sizecache.get(gen, self.fs.chunksize) | |
| blocknum = offset/self.chunksize | |
| blockoffs = offset - blocknum*self.chunksize | |
| # read a chunk if guessed blocknum and chunksize make sense | |
| if blocknum < len(self.chunkids): | |
| chunkdata = repo.get_chunk(self.chunkids[blocknum]) | |
| else: | |
| chunkdata = '' | |
| # check if chunkdata is of expected length | |
| validate = min(self.chunksize, self.metadata.st_size - blocknum*self.chunksize) | |
| if validate != len(chunkdata): | |
| if blocknum < len(self.chunkids)-1: | |
| # the length of all but last chunks is chunksize | |
| self.chunksize = len(chunkdata) | |
| else: | |
| # guessing failed, get the length of the first chunk | |
| self.chunksize = len(repo.get_chunk(self.chunkids[0])) | |
| chunkdata = None | |
| # save correct chunksize | |
| self.fs.sizecache[gen] = self.chunksize | |
| if not chunkdata: | |
| blocknum = offset/self.chunksize | |
| blockoffs = offset - blocknum*self.chunksize | |
| if self.lastblock == blocknum: | |
| chunkdata = self.lastdata | |
| else: | |
| chunkdata = repo.get_chunk(self.chunkids[blocknum]) | |
| output = [] | |
| while True: | |
| output.append(chunkdata[blockoffs:blockoffs+length]) | |
| readlength = len(chunkdata) - blockoffs | |
| if length > readlength and blocknum < len(self.chunkids)-1: | |
| length -= readlength | |
| blocknum += 1 | |
| blockoffs = 0 | |
| chunkdata = repo.get_chunk(self.chunkids[blocknum]) | |
| else: | |
| self.lastblock = blocknum | |
| self.lastdata = chunkdata | |
| break | |
| return ''.join(output) | |
| except (OSError, IOError), e: | |
| logging.debug('FUSE Expected exception') | |
| raise | |
| except: | |
| logging.exception('Unexpected exception') | |
| raise | |
| def release(self, flags): | |
| logging.debug('FUSE file release %d', flags) | |
| self.lastdata = None | |
| return 0 | |
| def fsync(self, isfsyncfile): | |
| logging.debug('FUSE file fsync') | |
| return 0 | |
| def flush(self): | |
| logging.debug('FUSE file flush') | |
| return 0 | |
| def ftruncate(self, size): | |
| logging.debug('FUSE file ftruncate %d', size) | |
| return 0 | |
| def lock(self, cmd, owner, **kw): | |
| logging.debug('FUSE file lock %s %s %s', repr(cmd), repr(owner), repr(kw)) | |
| raise IOError(errno.EOPNOTSUPP, 'Operation not supported') | |
| class ObnamFuse(fuse.Fuse): | |
| '''FUSE main class | |
| ''' | |
| MAX_METADATA_CACHE = 512 | |
| def root_refresh(self): | |
| logging.debug('FUSE root_refresh is called') | |
| if self.obnam.app.settings['viewmode'] == 'multiple': | |
| try: | |
| self.obnam.reopen() | |
| repo = self.obnam.repo | |
| generations = [gen for gen in repo.list_generations() | |
| if not repo.get_is_checkpoint(gen)] | |
| logging.debug('FUSE root_refresh found %d generations' % len(generations)) | |
| self.rootstat, self.rootlist = self.multiple_root_list(generations) | |
| self.metadatacache.clear() | |
| except: | |
| logging.exception('Unexpected exception') | |
| raise | |
| def get_metadata(self, path): | |
| #logging.debug('FUSE get_metadata(%s)', path) | |
| try: | |
| return self.metadatacache[path] | |
| except KeyError: | |
| if len(self.metadatacache) > self.MAX_METADATA_CACHE: | |
| self.metadatacache.clear() | |
| metadata = self.obnam.repo.get_metadata(*self.get_gen_path(path)) | |
| self.metadatacache[path] = metadata | |
| # FUSE does not allow negative timestamps, truncate to zero | |
| if metadata.st_atime_sec < 0: | |
| metadata.st_atime_sec = 0 | |
| if metadata.st_mtime_sec < 0: | |
| metadata.st_mtime_sec = 0 | |
| return metadata | |
| def get_stat(self, path): | |
| logging.debug('FUSE get_stat(%s)', path) | |
| metadata = self.get_metadata(path) | |
| st = fuse.Stat() | |
| st.st_mode = metadata.st_mode | |
| st.st_dev = metadata.st_dev | |
| st.st_nlink = metadata.st_nlink | |
| st.st_uid = metadata.st_uid | |
| st.st_gid = metadata.st_gid | |
| st.st_size = metadata.st_size | |
| st.st_atime = metadata.st_atime_sec | |
| st.st_mtime = metadata.st_mtime_sec | |
| st.st_ctime = st.st_mtime | |
| return st | |
| def single_root_list(self, gen): | |
| repo = self.obnam.repo | |
| mountroot = self.obnam.mountroot | |
| rootlist = {} | |
| for entry in repo.listdir(gen, mountroot): | |
| path = '/' + entry | |
| rootlist[path] = self.get_stat(path) | |
| rootstat = self.get_stat('/') | |
| return (rootstat, rootlist) | |
| def multiple_root_list(self, generations): | |
| repo = self.obnam.repo | |
| mountroot = self.obnam.mountroot | |
| rootlist = {} | |
| used_generations = [] | |
| for gen in generations: | |
| path = '/' + str(gen) | |
| try: | |
| genstat = self.get_stat(path) | |
| start, end = repo.get_generation_times(gen) | |
| genstat.st_ctime = genstat.st_mtime = end | |
| rootlist[path] = genstat | |
| used_generations.append(gen) | |
| except obnamlib.Error: | |
| pass | |
| if not used_generations: | |
| raise obnamlib.Error('No generations found for %s' % mountroot) | |
| latest = used_generations[-1] | |
| laststat = rootlist['/' + str(latest)] | |
| rootstat = fuse.Stat(**laststat.__dict__) | |
| laststat = fuse.Stat(target=str(latest), **laststat.__dict__) | |
| laststat.st_mode &= ~(stat.S_IFDIR | stat.S_IFREG) | |
| laststat.st_mode |= stat.S_IFLNK | |
| rootlist['/latest'] = laststat | |
| pidstat = fuse.Stat(**rootstat.__dict__) | |
| pidstat.st_mode = stat.S_IFREG | stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH | |
| rootlist['/.pid'] = pidstat | |
| return (rootstat, rootlist) | |
| def init_root(self): | |
| repo = self.obnam.repo | |
| mountroot = self.obnam.mountroot | |
| generations = self.obnam.app.settings['generation'] | |
| if self.obnam.app.settings['viewmode'] == 'single': | |
| if len(generations) != 1: | |
| raise obnamlib.Error( | |
| 'The single mode wants exactly one generation option') | |
| gen = repo.genspec(generations[0]) | |
| if mountroot == '/': | |
| self.get_gen_path = lambda path: (gen, path) | |
| else: | |
| self.get_gen_path = (lambda path : path == '/' | |
| and (gen, mountroot) | |
| or (gen, mountroot + path)) | |
| self.rootstat, self.rootlist = self.single_root_list(gen) | |
| logging.debug('FUSE single rootlist %s', repr(self.rootlist)) | |
| elif self.obnam.app.settings['viewmode'] == 'multiple': | |
| # we need the list of all real (non-checkpoint) generations | |
| if len(generations) == 1: | |
| generations = [gen for gen in repo.list_generations() | |
| if not repo.get_is_checkpoint(gen)] | |
| if mountroot == '/': | |
| def gen_path_0(path): | |
| if path.count('/') == 1: | |
| gen = path[1:] | |
| return (int(gen), mountroot) | |
| else: | |
| gen, repopath = path[1:].split('/', 1) | |
| return (int(gen), mountroot + repopath) | |
| self.get_gen_path = gen_path_0 | |
| else: | |
| def gen_path_n(path): | |
| if path.count('/') == 1: | |
| gen = path[1:] | |
| return (int(gen), mountroot) | |
| else: | |
| gen, repopath = path[1:].split('/', 1) | |
| return (int(gen), mountroot + '/' + repopath) | |
| self.get_gen_path = gen_path_n | |
| self.rootstat, self.rootlist = self.multiple_root_list(generations) | |
| logging.debug('FUSE multiple rootlist %s', repr(self.rootlist)) | |
| else: | |
| raise obnamlib.Error('Unknown value for viewmode') | |
| def __init__(self, *args, **kw): | |
| self.obnam = kw['obnam'] | |
| ObnamFuseFile.fs = self | |
| self.file_class = ObnamFuseFile | |
| self.metadatacache = {} | |
| self.chunksize = self.obnam.app.settings['chunk-size'] | |
| self.sizecache = {} | |
| self.rootlist = None | |
| self.rootstat = None | |
| self.init_root() | |
| fuse.Fuse.__init__(self, *args, **kw) | |
| def getattr(self, path): | |
| try: | |
| if path.count('/') == 1: | |
| if path == '/': | |
| return self.rootstat | |
| elif path in self.rootlist: | |
| return self.rootlist[path] | |
| else: | |
| raise obnamlib.Error('ENOENT') | |
| else: | |
| return self.get_stat(path) | |
| except obnamlib.Error: | |
| raise IOError(errno.ENOENT, 'No such file or directory') | |
| except: | |
| logging.error('Unexpected exception', exc_info=True) | |
| raise | |
| def readdir(self, path, fh): | |
| logging.debug('FUSE readdir(%s, %s)', path, repr(fh)) | |
| try: | |
| if path == '/': | |
| listdir = [x[1:] for x in self.rootlist.keys()] | |
| else: | |
| listdir = self.obnam.repo.listdir(*self.get_gen_path(path)) | |
| return [fuse.Direntry(name) for name in ['.', '..'] + listdir] | |
| except obnamlib.Error: | |
| raise IOError(errno.EINVAL, 'Invalid argument') | |
| except: | |
| logging.error('Unexpected exception', exc_info=True) | |
| raise | |
| def readlink(self, path): | |
| try: | |
| statdata = self.rootlist.get(path) | |
| if statdata and hasattr(statdata, 'target'): | |
| return statdata.target | |
| metadata = self.get_metadata(path) | |
| if metadata.islink(): | |
| return metadata.target | |
| else: | |
| raise IOError(errno.EINVAL, 'Invalid argument') | |
| except obnamlib.Error: | |
| raise IOError(errno.ENOENT, 'No such file or directory') | |
| except: | |
| logging.error('Unexpected exception', exc_info=True) | |
| raise | |
| def statfs(self): | |
| logging.debug('FUSE statfs') | |
| try: | |
| repo = self.obnam.repo | |
| if self.obnam.app.settings['viewmode'] == 'multiple': | |
| blocks = sum(repo.client.get_generation_data(gen) | |
| for gen in repo.list_generations()) | |
| files = sum(repo.client.get_generation_file_count(gen) | |
| for gen in repo.list_generations()) | |
| else: | |
| gen = self.get_gen_path('/')[0] | |
| blocks = repo.client.get_generation_data(gen) | |
| files = repo.client.get_generation_file_count(gen) | |
| stv = fuse.StatVfs() | |
| stv.f_bsize = 65536 | |
| stv.f_frsize = 0 | |
| stv.f_blocks = blocks/65536 | |
| stv.f_bfree = 0 | |
| stv.f_bavail = 0 | |
| stv.f_files = files | |
| stv.f_ffree = 0 | |
| stv.f_favail = 0 | |
| stv.f_flag = 0 | |
| stv.f_namemax = 255 | |
| #raise OSError(errno.ENOSYS, 'Unimplemented') | |
| return stv | |
| except: | |
| logging.error('Unexpected exception', exc_info=True) | |
| raise | |
| def getxattr(self, path, name, size): | |
| logging.debug('FUSE getxattr %s %s %d', path, name, size) | |
| try: | |
| try: | |
| metadata = self.get_metadata(path) | |
| except ValueError: | |
| return 0 | |
| if not metadata.xattr: | |
| return 0 | |
| blob = metadata.xattr | |
| sizesize = struct.calcsize('!Q') | |
| name_blob_size = struct.unpack('!Q', blob[:sizesize])[0] | |
| name_blob = blob[sizesize : sizesize + name_blob_size] | |
| name_list = name_blob.split('\0')[:-1] | |
| if name in name_list: | |
| value_blob = blob[sizesize + name_blob_size : ] | |
| idx = name_list.index(name) | |
| fmt = '!' + 'Q' * len(name_list) | |
| lengths_size = sizesize * len(name_list) | |
| lengths_list = struct.unpack(fmt, value_blob[:lengths_size]) | |
| if size == 0: | |
| return lengths_list[idx] | |
| pos = lengths_size + sum(lengths_list[:idx]) | |
| value = value_blob[pos:pos + lengths_list[idx]] | |
| return value | |
| except obnamlib.Error: | |
| raise IOError(errno.ENOENT, 'No such file or directory') | |
| except: | |
| logging.error('Unexpected exception', exc_info=True) | |
| raise | |
| def listxattr(self, path, size): | |
| logging.debug('FUSE listxattr %s %d', path, size) | |
| try: | |
| metadata = self.get_metadata(path) | |
| if not metadata.xattr: | |
| return 0 | |
| blob = metadata.xattr | |
| sizesize = struct.calcsize('!Q') | |
| name_blob_size = struct.unpack('!Q', blob[:sizesize])[0] | |
| if size == 0: | |
| return name_blob_size | |
| name_blob = blob[sizesize : sizesize + name_blob_size] | |
| return name_blob.split('\0')[:-1] | |
| except obnamlib.Error: | |
| raise IOError(errno.ENOENT, 'No such file or directory') | |
| except: | |
| logging.error('Unexpected exception', exc_info=True) | |
| raise | |
| def fsync(self, path, isFsyncFile): | |
| return 0 | |
| def chmod(self, path, mode): | |
| raise IOError(errno.EROFS, 'Read only filesystem') | |
| def chown(self, path, uid, gid): | |
| raise IOError(errno.EROFS, 'Read only filesystem') | |
| def link(self, targetPath, linkPath): | |
| raise IOError(errno.EROFS, 'Read only filesystem') | |
| def mkdir(self, path, mode): | |
| raise IOError(errno.EROFS, 'Read only filesystem') | |
| def mknod(self, path, mode, dev): | |
| raise IOError(errno.EROFS, 'Read only filesystem') | |
| def rename(self, oldPath, newPath): | |
| raise IOError(errno.EROFS, 'Read only filesystem') | |
| def rmdir(self, path): | |
| raise IOError(errno.EROFS, 'Read only filesystem') | |
| def symlink(self, targetPath, linkPath): | |
| raise IOError(errno.EROFS, 'Read only filesystem') | |
| def truncate(self, path, size): | |
| raise IOError(errno.EROFS, 'Read only filesystem') | |
| def unlink(self, path): | |
| raise IOError(errno.EROFS, 'Read only filesystem') | |
| def utime(self, path, times): | |
| raise IOError(errno.EROFS, 'Read only filesystem') | |
| def write(self, path, buf, offset): | |
| raise IOError(errno.EROFS, 'Read only filesystem') | |
| def setxattr(self, path, name, val, flags): | |
| raise IOError(errno.EROFS, 'Read only filesystem') | |
| def removexattr(self, path, name): | |
| raise IOError(errno.EROFS, 'Read only filesystem') | |
| class MountPlugin(obnamlib.ObnamPlugin): | |
| '''Mount backup repository as a user-space filesystem. | |
| At the momemnt only a specific generation can be mounted | |
| ''' | |
| def enable(self): | |
| mount_group = obnamlib.option_group['mount'] = 'Mounting with FUSE' | |
| self.app.add_subcommand('mount', self.mount, | |
| arg_synopsis='[ROOT]') | |
| self.app.settings.choice(['viewmode'], | |
| ['single', 'multiple'], | |
| '"single" directly mount specified generation, ' | |
| '"multiple" mount all generations as separate directories', | |
| metavar='MODE', | |
| group=mount_group) | |
| self.app.settings.string_list(['fuse-opt'], | |
| 'options to pass directly to Fuse', | |
| metavar='FUSE', group=mount_group) | |
| def mount(self, args): | |
| '''Mount a backup repository as a FUSE filesystem. | |
| This subcommand allows you to access backups in an Obnam | |
| backup repository as normal files and directories. Each | |
| backed up file or directory can be viewed directly, using | |
| a graphical file manager or command line tools. | |
| Example: To mount your backup repository: | |
| mkdir my-fuse | |
| obnam mount --viewmode multiple --to my-fuse | |
| You can then access the backup using commands such as these: | |
| ls -l my-fuse | |
| ls -l my-fuse/latest | |
| diff -u my-fuse/latest/home/liw/README ~/README | |
| You can also restore files by copying them from the | |
| my-fuse directory: | |
| cp -a my-fuse/12765/Maildir ~/Maildir.restored | |
| To un-mount: | |
| fusermount -u my-fuse | |
| ''' | |
| if not hasattr(fuse, 'fuse_python_api'): | |
| raise obnamlib.Error('Failed to load module "fuse", ' | |
| 'try installing python-fuse') | |
| self.app.settings.require('repository') | |
| self.app.settings.require('client-name') | |
| self.app.settings.require('to') | |
| self.cwd = os.getcwd() | |
| self.repo = self.app.open_repository() | |
| self.repo.open_client(self.app.settings['client-name']) | |
| self.mountroot = (['/'] + self.app.settings['root'] + args)[-1] | |
| if self.mountroot != '/': | |
| self.mountroot = self.mountroot.rstrip('/') | |
| logging.debug('FUSE Mounting %s@%s:%s to %s', self.app.settings['client-name'], | |
| self.app.settings['generation'], | |
| self.mountroot, self.app.settings['to']) | |
| try: | |
| ObnamFuseOptParse.obnam = self | |
| fs = ObnamFuse(obnam=self, parser_class=ObnamFuseOptParse) | |
| fs.flags = 0 | |
| fs.multithreaded = 0 | |
| fs.parse() | |
| fs.main() | |
| except fuse.FuseError, e: | |
| raise obnamlib.Error(repr(e)) | |
| self.repo.fs.close() | |
| def reopen(self): | |
| try: | |
| os.chdir(self.cwd) | |
| except OSError: | |
| pass | |
| self.repo.fs.close() | |
| self.repo = self.app.open_repository() | |
| self.repo.open_client(self.app.settings['client-name']) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
This comment has been minimized.
Last single mode version is https://gist.github.com/4547295/987b453cbcc37710959abdb392558201ec3e44e8