omegaml support info collector

omx_getinfo.py

def getinfo(om, models=None, datasets=None, scripts=None, jobs=None, outfile=None, tryget=False):
    """
    get info relevant to provide support
    
    Usage:
        import omegaml as om 
        
        # to get a printed report
        print(getinfo(om, ...))
        
        # to get a pickle file to send to support
        getinfo(om, ..., outfile='/path/to/info.pkl')
        
    Args:
        om (Omega): the omega instance
        models (str|list): name or list of names of models you want to collect info about
        datasets (str|list): name or list of names of datasets you want to collect info about
        scripts (str|list): name or list of names of scripts you want to collect info about
        jobs (str|list): name or list of names of jobs you want to collect info about
        
        
    What information is collected?
        the resulting dict contains the following keys:
        
        'omega': the omegaml, pymongo, mongoengine package versions
        'defaults': the om.defaults settings, only uppercase settings are collected. secrets and passwords are masked
        'sys': the contents of sys.path, sys.version, sys.platform 
        'datasets': metadata about each dataset (as given by datasets=)
        'models': metadata about each model (as given by models=)
        'jobs': metadata about each job (as given by jobs=)
        'scripts': metadata about each script (as given by scripts=)
        
        The metadata collect for each of datasets, models, jobs, scripts is
        
        'attributes': metadata.attributes 
        'kind_meta': metadata.kind_meta
        'gridfile': the id of the gridfile, mongodb, collection, alias name and whether any data can be read 
            (not the data itself)
    """
    from collections import defaultdict
    from pprint import pformat
    import pickle
    import sys
    import re
    import pymongo
    import mongoengine
    from omegaml._version import version
    def tryor(f):
        try:
            result = f()
        except Exception as e:
            result = repr(e)
        return result
    clean = lambda s: re.sub(r':\/\/(.*?)(\@)', r'://***@', str(s))
    mask_K = ['KEY', 'SECRET', 'PASSWORD', 'TOKEN', 'PASSW']
    mask_E = ['ROUTING_KEY']
    anyin = lambda k, kk: any(k_ in k for k_ in kk)
    mask = lambda d: {k: '*****' if (anyin(k, mask_K) and not anyin(k, mask_E)) else clean(v) for k, v in d.items() }
    makelist = lambda v: (v if isinstance(v, (list, tuple)) else [v]) if v is not None else []
    models = makelist(models)
    datasets = makelist(datasets)
    scripts = makelist(scripts)
    jobs = makelist(jobs)
    info = defaultdict(dict)
    if hasattr(om, 'setup'):
        om = om.setup()
    info['defaults'] = mask({ k: clean(getattr(om.defaults, k)) for k in dir(om.defaults) if k.isupper()})
    if isinstance(models, str):
        models = [models]
    if isinstance(datasets, str):
        datasets = [datasets]
    stores = ('models', 'datasets', 'scripts', 'jobs')
    names = (models, datasets, scripts, jobs)
    for s, ns  in zip(stores, names):
        for n in ns:
            store = getattr(om, s)
            meta = store.metadata(n)
            info[s]['mixins'] = list(str(cls) for cls in store.__class__.mro())
            info[s]['mongo_url'] = tryor(lambda : clean(store.mongo_url))
            info[s]['store._dbalias'] = str(tryor(lambda : store._dbalias))
            info[s]['store._Metadata._meta'] = str(tryor(lambda : store._Metadata._meta))
            info[s][n] = {}
            info[s][n]['meta'] = repr(meta)
            if meta is not None:
                info[s][n]['backend'] = tryor(lambda : repr(store.get_backend_bykind(meta.kind)))
                info[s][n]['attributes'] = meta.attributes
                info[s][n]['kind_meta'] = meta.kind_meta
                info[s][n]['metadata._meta'] = meta._meta
                info[s][n]['collection_ref'] = str(tryor(lambda : meta.collection))
                info[s][n]['collection'] = str(tryor(lambda : store.collection(n)))
                info[s][n]['collection.name'] = str(tryor(lambda : store.collection(n).name))
                info[s][n]['gridfile'] = str(tryor(lambda : meta.gridfile))
                info[s][n]['gridfile.read'] = str(tryor(lambda : 'ok' if meta.gridfile.read() is not None else 'None'))                
                info[s][n]['gridfile.name'] = str(tryor(lambda : meta.gridfile.name))                
                info[s][n]['gridfile.fs.collection'] = str(tryor(lambda : meta.gridfile.fs._GridFS__collection))             
                info[s][n]['gridfile.db_alias'] = str(tryor(lambda : meta.gridfile.db_alias))             
                if tryget:
                    info[s][n]['get'] = str(tryor(lambda : 'ok' if store.get(n) is not None else 'None'))
                    info[s][n]['get_lazy'] = str(tryor(lambda : 'ok' if store.getl(n) is not None else 'None'))
    info['sys']['path'] = sys.path
    info['sys']['version'] = sys.version
    info['sys']['platform'] = sys.platform
    info['omega']['version'] = version
    info['omega']['pymongo'] = pymongo.version
    info['omega']['mongoengine'] = mongoengine.__version__
    info['omega']['runtime'] = str(tryor(lambda : clean(om.runtime)))
    info['omega']['runtime.celeryapp.conf'] = mask(tryor(lambda : dict(om.runtime.celeryapp.conf)))
    info['omega']['runtime.ping'] = str(tryor(lambda : timeout(om.runtime.ping, seconds=10)))
    if not outfile:
        result = dict(info)
    else:
        if isinstance(outfile, str):
            fout = open(outfile, 'w')
        else:
            fout = outfile
        fout.write(pformat(info))
        result = fout.name
        fout.close()
    return result
  
def timeout(func, args=(), kwargs={}, seconds=1, default='timed out'):
    # adopted from https://stackoverflow.com/a/13821695
    import signal
    timeout_duration = seconds
    class TimeoutError(Exception):
        pass
    def handler(signum, frame):
        raise TimeoutError()
    signal.signal(signal.SIGALRM, handler) 
    signal.alarm(timeout_duration)
    try:
        result = func(*args, **kwargs)
    except TimeoutError as exc:
        result = default
    finally:
        signal.alarm(0)
    return result
    
if __name__ == '__main__':
    import argparse
    from pprint import pprint
    import omegaml as om 
    
    parser = argparse.ArgumentParser(description='omegaml getinfo')
    parser.add_argument('--models', type=str, default='', help='model[,...]')
    parser.add_argument('--datasets', type=str, default='', help='dataset[,...]')
    parser.add_argument('--scripts', type=str, default='', help='scripts[,...]')
    parser.add_argument('--jobs', type=str, default='', help='jobs[,...]')
    parser.add_argument('--output', type=str, help='/path/to/file.getinfo')
    parser.add_argument('--explain', action='store_true', help='explain what getinfo does')
    args = parser.parse_args()
    models = args.models.split(',')
    datasets = args.datasets.split(',')
    scripts = args.scripts.split(',')
    jobs = args.jobs.split(',')
    if args.explain:
        help(getinfo)
        exit(0)
    print("getinfo is collecting information...")
    result = getinfo(om, models=models, datasets=datasets, jobs=jobs, scripts=scripts, outfile=args.output)
    pprint(result)

omx_getinfo.sh

#!/bin/bash
pip install -U getgist > getinfo.log
getgist -y omegaml omx_getinfo.py >> getinfo.log
python -m omx_getinfo --output getinfo.txt $* 
cat getinfo.log >> getinfo.txt
rm getinfo.log
echo "please send the file getinfo.txt to support@omegaml.io or as indiciated by omega-ml support staff"

On direction of omegaml support,

1. open a shell to your python environment (e.g. in Jupyter, open a Terminal)
2. create the omx_getinfo.sh file with above contents
3. run chmod +x omx_getinfo.sh
4. run ./omx_getinfo.sh

Then send the file getinfo.txt to omegaml support. The file contains relevant information for your support agent to assist with your problem.

Note that step 4 may need additional options.

• If you want help with a particular model, add --models <name>
• If you want help with a particular dataset, add --datasets <name>
• If you want help with a particular script, add --scripts <name>
• If you want help with a particular job (notebook), add --jobs <name>

To understand what information getinfo is collecting, run omx_getinfo.sh --explain. You may also look at the getinfo.txt file generated by the tool, all information is shown in plain text.