Last active
May 30, 2020 09:29
-
-
Save omegaml/b305543c83367c6714f0c77f3dafc5ff to your computer and use it in GitHub Desktop.
omegaml support info collector
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
def getinfo(om, models=None, datasets=None, scripts=None, jobs=None, outfile=None, tryget=False): | |
""" | |
get info relevant to provide support | |
Usage: | |
import omegaml as om | |
# to get a printed report | |
print(getinfo(om, ...)) | |
# to get a pickle file to send to support | |
getinfo(om, ..., outfile='/path/to/info.pkl') | |
Args: | |
om (Omega): the omega instance | |
models (str|list): name or list of names of models you want to collect info about | |
datasets (str|list): name or list of names of datasets you want to collect info about | |
scripts (str|list): name or list of names of scripts you want to collect info about | |
jobs (str|list): name or list of names of jobs you want to collect info about | |
What information is collected? | |
the resulting dict contains the following keys: | |
'omega': the omegaml, pymongo, mongoengine package versions | |
'defaults': the om.defaults settings, only uppercase settings are collected. secrets and passwords are masked | |
'sys': the contents of sys.path, sys.version, sys.platform | |
'datasets': metadata about each dataset (as given by datasets=) | |
'models': metadata about each model (as given by models=) | |
'jobs': metadata about each job (as given by jobs=) | |
'scripts': metadata about each script (as given by scripts=) | |
The metadata collect for each of datasets, models, jobs, scripts is | |
'attributes': metadata.attributes | |
'kind_meta': metadata.kind_meta | |
'gridfile': the id of the gridfile, mongodb, collection, alias name and whether any data can be read | |
(not the data itself) | |
""" | |
from collections import defaultdict | |
from pprint import pformat | |
import pickle | |
import sys | |
import re | |
import pymongo | |
import mongoengine | |
from omegaml._version import version | |
def tryor(f): | |
try: | |
result = f() | |
except Exception as e: | |
result = repr(e) | |
return result | |
clean = lambda s: re.sub(r':\/\/(.*?)(\@)', r'://***@', str(s)) | |
mask_K = ['KEY', 'SECRET', 'PASSWORD', 'TOKEN', 'PASSW'] | |
mask_E = ['ROUTING_KEY'] | |
anyin = lambda k, kk: any(k_ in k for k_ in kk) | |
mask = lambda d: {k: '*****' if (anyin(k, mask_K) and not anyin(k, mask_E)) else clean(v) for k, v in d.items() } | |
makelist = lambda v: (v if isinstance(v, (list, tuple)) else [v]) if v is not None else [] | |
models = makelist(models) | |
datasets = makelist(datasets) | |
scripts = makelist(scripts) | |
jobs = makelist(jobs) | |
info = defaultdict(dict) | |
if hasattr(om, 'setup'): | |
om = om.setup() | |
info['defaults'] = mask({ k: clean(getattr(om.defaults, k)) for k in dir(om.defaults) if k.isupper()}) | |
if isinstance(models, str): | |
models = [models] | |
if isinstance(datasets, str): | |
datasets = [datasets] | |
stores = ('models', 'datasets', 'scripts', 'jobs') | |
names = (models, datasets, scripts, jobs) | |
for s, ns in zip(stores, names): | |
for n in ns: | |
store = getattr(om, s) | |
meta = store.metadata(n) | |
info[s]['mixins'] = list(str(cls) for cls in store.__class__.mro()) | |
info[s]['mongo_url'] = tryor(lambda : clean(store.mongo_url)) | |
info[s]['store._dbalias'] = str(tryor(lambda : store._dbalias)) | |
info[s]['store._Metadata._meta'] = str(tryor(lambda : store._Metadata._meta)) | |
info[s][n] = {} | |
info[s][n]['meta'] = repr(meta) | |
if meta is not None: | |
info[s][n]['backend'] = tryor(lambda : repr(store.get_backend_bykind(meta.kind))) | |
info[s][n]['attributes'] = meta.attributes | |
info[s][n]['kind_meta'] = meta.kind_meta | |
info[s][n]['metadata._meta'] = meta._meta | |
info[s][n]['collection_ref'] = str(tryor(lambda : meta.collection)) | |
info[s][n]['collection'] = str(tryor(lambda : store.collection(n))) | |
info[s][n]['collection.name'] = str(tryor(lambda : store.collection(n).name)) | |
info[s][n]['gridfile'] = str(tryor(lambda : meta.gridfile)) | |
info[s][n]['gridfile.read'] = str(tryor(lambda : 'ok' if meta.gridfile.read() is not None else 'None')) | |
info[s][n]['gridfile.name'] = str(tryor(lambda : meta.gridfile.name)) | |
info[s][n]['gridfile.fs.collection'] = str(tryor(lambda : meta.gridfile.fs._GridFS__collection)) | |
info[s][n]['gridfile.db_alias'] = str(tryor(lambda : meta.gridfile.db_alias)) | |
if tryget: | |
info[s][n]['get'] = str(tryor(lambda : 'ok' if store.get(n) is not None else 'None')) | |
info[s][n]['get_lazy'] = str(tryor(lambda : 'ok' if store.getl(n) is not None else 'None')) | |
info['sys']['path'] = sys.path | |
info['sys']['version'] = sys.version | |
info['sys']['platform'] = sys.platform | |
info['omega']['version'] = version | |
info['omega']['pymongo'] = pymongo.version | |
info['omega']['mongoengine'] = mongoengine.__version__ | |
info['omega']['runtime'] = str(tryor(lambda : clean(om.runtime))) | |
info['omega']['runtime.celeryapp.conf'] = mask(tryor(lambda : dict(om.runtime.celeryapp.conf))) | |
info['omega']['runtime.ping'] = str(tryor(lambda : timeout(om.runtime.ping, seconds=10))) | |
if not outfile: | |
result = dict(info) | |
else: | |
if isinstance(outfile, str): | |
fout = open(outfile, 'w') | |
else: | |
fout = outfile | |
fout.write(pformat(info)) | |
result = fout.name | |
fout.close() | |
return result | |
def timeout(func, args=(), kwargs={}, seconds=1, default='timed out'): | |
# adopted from https://stackoverflow.com/a/13821695 | |
import signal | |
timeout_duration = seconds | |
class TimeoutError(Exception): | |
pass | |
def handler(signum, frame): | |
raise TimeoutError() | |
signal.signal(signal.SIGALRM, handler) | |
signal.alarm(timeout_duration) | |
try: | |
result = func(*args, **kwargs) | |
except TimeoutError as exc: | |
result = default | |
finally: | |
signal.alarm(0) | |
return result | |
if __name__ == '__main__': | |
import argparse | |
from pprint import pprint | |
import omegaml as om | |
parser = argparse.ArgumentParser(description='omegaml getinfo') | |
parser.add_argument('--models', type=str, default='', help='model[,...]') | |
parser.add_argument('--datasets', type=str, default='', help='dataset[,...]') | |
parser.add_argument('--scripts', type=str, default='', help='scripts[,...]') | |
parser.add_argument('--jobs', type=str, default='', help='jobs[,...]') | |
parser.add_argument('--output', type=str, help='/path/to/file.getinfo') | |
parser.add_argument('--explain', action='store_true', help='explain what getinfo does') | |
args = parser.parse_args() | |
models = args.models.split(',') | |
datasets = args.datasets.split(',') | |
scripts = args.scripts.split(',') | |
jobs = args.jobs.split(',') | |
if args.explain: | |
help(getinfo) | |
exit(0) | |
print("getinfo is collecting information...") | |
result = getinfo(om, models=models, datasets=datasets, jobs=jobs, scripts=scripts, outfile=args.output) | |
pprint(result) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/bash | |
pip install -U getgist > getinfo.log | |
getgist -y omegaml omx_getinfo.py >> getinfo.log | |
python -m omx_getinfo --output getinfo.txt $* | |
cat getinfo.log >> getinfo.txt | |
rm getinfo.log | |
echo "please send the file getinfo.txt to support@omegaml.io or as indiciated by omega-ml support staff" |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
On direction of omegaml support,
omx_getinfo.sh
file with above contentschmod +x omx_getinfo.sh
./omx_getinfo.sh
Then send the file
getinfo.txt
to omegaml support. The file contains relevant information for your support agent to assist with your problem.Note that step 4 may need additional options.
--models <name>
--datasets <name>
--scripts <name>
--jobs <name>
To understand what information getinfo is collecting, run
omx_getinfo.sh --explain
. You may also look at thegetinfo.txt
file generated by the tool, all information is shown in plain text.