List all python3 packages and versions present in the AWS lambda environment

lambda.py

#!/usr/bin/env python3.7
'''
List all of the installed packages in the environment and write them
to the S3 bucket.
'''
import http
import json
import importlib
import os
import pkgutil
import re
import sys

import boto3

try:
    S3_BUCKET = os.environ['S3_BUCKET']
except KeyError:
    S3_BUCKET = None
PYVER = f'python-{sys.version_info[0]}.{sys.version_info[1]}.{sys.version_info[2]}'
VERSION_PATTERN = re.compile(r'^.*version.*$', re.IGNORECASE)


def handler(_event: dict, _context: dict) -> dict:
    '''
    Generate the list of installed packages.
    '''
    pkgs = []
    ##names = [f'{m.name}' for m in pkgutil.iter_modules() if m.ispkg]
    names = [f'{m.name}' for m in pkgutil.iter_modules()]
    names += list(sys.builtin_module_names)
    num_native = 0
    num_installed = 0
    for pkg in names:
        version = get_pkg_version(pkg)
        pkgs.append(f'{pkg}=={version}')
        if 'python' in version:
            num_native += 1
        else:
            num_installed += 1
    body = {'total': len(pkgs),
            'num_native': num_native,
            'num_installed': num_installed,
            'pkgs': sorted(pkgs, key=lambda s: s.lstrip('_').lower())}
    content = '\n'.join([f'# python: {i}' for i in sys.version.split('\n')])
    content += '\n'
    content += f'# num_native:    {num_native:>4}\n'
    content += f'# num_installed: {num_installed:>4}\n'
    content += f'# total:         {len(pkgs):>4}\n'
    content += '\n'.join(sorted(pkgs, key=lambda s: s.lstrip('_').lower())) + '\n'
    if not S3_BUCKET:
        ofp = sys.stdout
        ofp.write(content)
    else:
        # To view the data, highlighting the non-native packages.
        # $ aws s3 cp s3://jlinoff-list-packages/python-3.7.3.txt /dev/stdout | \
        #      grep -v 'download: s3://' | \
        #      cat -n | \
        #      colorize -c bold+green '^.+==\d.*$'
        obj = boto3.resource('s3')
        path = f'{PYVER}.txt'
        obj.Object(S3_BUCKET, path).put(Body=content)
        print(f'Wrote package info to "s3://{S3_BUCKET}/{path}".')
    return make_status(200, body)


def get_pkg_version(pkg: str) -> str:
    '''
    Figure out the package version.

    This is definitely a heuristic operation.

    The idea is to look for strings like __version__,
    version or VERSION.
    '''
    default_version = PYVER
    if pkg in ['sys', 'this', 'antigravity']:
        return default_version
    try:
        imp = importlib.import_module(pkg)
        versions = []
        for attr in dir(imp):
            if VERSION_PATTERN.match(attr):
                obj = getattr(imp, attr)
                if isinstance(obj, str):
                    versions.append(attr)

        versions = prune_versions(versions)

        if not versions:
            version = default_version
        elif len(versions) == 1:
            version = getattr(imp, versions[0])
        else:
            # multiple versions, pick the first one.
            version = getattr(imp, versions[0])
    except ModuleNotFoundError:
        version = 'error:module'

    return version


def prune_versions(versions: list) -> list:
    '''
    Prune the versions.
    '''
    # In cases where there are multiple versions,
    # choose the one we want with precedence.
    if len(versions) > 1:
        done = False
        for preferred in ['__version__', 'version', 'VERSION']:
            for value in versions:
                if preferred == value:
                    versions = [value]
                    done = True
                    break
            if done:
                break
    return versions


def make_status(code, msg='') -> dict:
    '''
    Make status codes in a uniform way.
    '''
    # This format was recommended by AWS.
    if not msg:
        try:
            msg = http.client.responses[code]
        except KeyError:
            msg = f'invalid key {code}'

    payload = {
        'isBase64Encoded': 'false',
        'statusCode': code,
        'headers': {'Content-Type': 'application/json'},
        'body': json.dumps(msg)
    }
    return payload


if __name__ == '__main__':
    print(json.dumps(handler({}, {}), indent=2))

python-3.7.3

# python: 3.7.3 (default, Apr 18 2019, 11:13:26) 
# python: [GCC 4.8.3 20140911 (Red Hat 4.8.3-9)]
# num_native:     274
# num_installed:   46
# total:          320
abc==python-3.7.3
_abc==python-3.7.3
aifc==python-3.7.3
antigravity==python-3.7.3
argparse==1.1
array==python-3.7.3
ast==python-3.7.3
_ast==python-3.7.3
asynchat==python-3.7.3
asyncio==python-3.7.3
_asyncio==python-3.7.3
asyncore==python-3.7.3
atexit==python-3.7.3
audioop==python-3.7.3
base64==python-3.7.3
bdb==python-3.7.3
binascii==python-3.7.3
binhex==python-3.7.3
bisect==python-3.7.3
_bisect==python-3.7.3
_blake2==python-3.7.3
_bootlocale==python-3.7.3
bootstrap==python-3.7.3
boto3==1.9.42
botocore==1.12.42
builtins==python-3.7.3
bz2==python-3.7.3
_bz2==python-3.7.3
calendar==python-3.7.3
cgi==2.6
cgitb==python-3.7.3
chunk==python-3.7.3
cmath==python-3.7.3
cmd==python-3.7.3
code==python-3.7.3
codecs==python-3.7.3
_codecs==python-3.7.3
_codecs_cn==python-3.7.3
_codecs_hk==python-3.7.3
_codecs_iso2022==python-3.7.3
_codecs_jp==python-3.7.3
_codecs_kr==python-3.7.3
_codecs_tw==python-3.7.3
codeop==python-3.7.3
collections==python-3.7.3
_collections==python-3.7.3
_collections_abc==python-3.7.3
colorsys==python-3.7.3
_compat_pickle==python-3.7.3
compileall==python-3.7.3
_compression==python-3.7.3
concurrent==python-3.7.3
configparser==python-3.7.3
contextlib==python-3.7.3
contextvars==python-3.7.3
_contextvars==python-3.7.3
copy==python-3.7.3
copyreg==python-3.7.3
cProfile==python-3.7.3
crypt==python-3.7.3
_crypt==python-3.7.3
csv==1.0
_csv==1.0
ctypes==1.1.0
_ctypes==1.1.0
_ctypes_test==python-3.7.3
curses==python-3.7.3
_curses==python-3.7.3
_curses_panel==2.1
dataclasses==python-3.7.3
datetime==python-3.7.3
_datetime==python-3.7.3
dateutil==2.7.5
dbm==python-3.7.3
_dbm==python-3.7.3
decimal==1.70
_decimal==1.70
difflib==python-3.7.3
dis==python-3.7.3
distutils==3.7.3
doctest==python-3.7.3
docutils==0.14
_dummy_thread==python-3.7.3
dummy_threading==python-3.7.3
easy_install==python-3.7.3
_elementtree==python-3.7.3
email==python-3.7.3
encodings==python-3.7.3
ensurepip==19.0.3
enum==python-3.7.3
errno==python-3.7.3
faulthandler==python-3.7.3
fcntl==python-3.7.3
filecmp==python-3.7.3
fileinput==python-3.7.3
fnmatch==python-3.7.3
formatter==python-3.7.3
fractions==python-3.7.3
ftplib==python-3.7.3
functools==python-3.7.3
_functools==python-3.7.3
__future__==python-3.7.3
gc==python-3.7.3
_gdbm==python-3.7.3
genericpath==python-3.7.3
getopt==python-3.7.3
getpass==python-3.7.3
gettext==python-3.7.3
glob==python-3.7.3
grp==python-3.7.3
gzip==python-3.7.3
hashlib==python-3.7.3
_hashlib==python-3.7.3
heapq==python-3.7.3
_heapq==python-3.7.3
hmac==python-3.7.3
html==python-3.7.3
http==python-3.7.3
idlelib==python-3.7.3
imaplib==2.58
imghdr==python-3.7.3
imp==python-3.7.3
_imp==python-3.7.3
importlib==python-3.7.3
inspect==python-3.7.3
io==python-3.7.3
_io==python-3.7.3
ipaddress==1.0
itertools==python-3.7.3
jmespath==0.9.3
json==2.0.9
_json==python-3.7.3
keyword==python-3.7.3
lambda==python-3.7.3
lambda_runtime_client==python-3.7.3
lambda_runtime_exception==python-3.7.3
lambda_runtime_marshaller==python-3.7.3
lib2to3==python-3.7.3
linecache==python-3.7.3
locale==python-3.7.3
_locale==python-3.7.3
logging==0.5.1.2
_lsprof==python-3.7.3
lzma==python-3.7.3
_lzma==python-3.7.3
macpath==python-3.7.3
mailbox==python-3.7.3
mailcap==python-3.7.3
_markupbase==python-3.7.3
marshal==python-3.7.3
math==python-3.7.3
_md5==python-3.7.3
mimetypes==python-3.7.3
mmap==python-3.7.3
modulefinder==python-3.7.3
_multibytecodec==python-3.7.3
multiprocessing==python-3.7.3
_multiprocessing==python-3.7.3
netrc==python-3.7.3
nis==python-3.7.3
nntplib==python-3.7.3
ntpath==python-3.7.3
nturl2path==python-3.7.3
numbers==python-3.7.3
opcode==python-3.7.3
_opcode==python-3.7.3
operator==python-3.7.3
_operator==python-3.7.3
optparse==1.5.3
os==python-3.7.3
ossaudiodev==python-3.7.3
_osx_support==python-3.7.3
parser==0.5
pathlib==python-3.7.3
pdb==python-3.7.3
pickle==4.0
_pickle==python-3.7.3
pickletools==python-3.7.3
pip==19.0.3
pipes==python-3.7.3
pkg_resources==python-3.7.3
pkgutil==python-3.7.3
platform==1.0.8
plistlib==python-3.7.3
poplib==python-3.7.3
posix==python-3.7.3
posixpath==python-3.7.3
_posixsubprocess==python-3.7.3
pprint==python-3.7.3
profile==python-3.7.3
pstats==python-3.7.3
pty==python-3.7.3
pwd==python-3.7.3
_py_abc==python-3.7.3
py_compile==python-3.7.3
pyclbr==python-3.7.3
_pydecimal==1.70
pydoc==python-3.7.3
pydoc_data==python-3.7.3
pyexpat==expat_2.2.6
_pyio==python-3.7.3
queue==python-3.7.3
_queue==python-3.7.3
quopri==python-3.7.3
random==python-3.7.3
_random==python-3.7.3
re==2.2.1
readline==6.2
reprlib==python-3.7.3
resource==python-3.7.3
rlcompleter==python-3.7.3
runpy==python-3.7.3
s3transfer==0.1.13
sched==python-3.7.3
secrets==python-3.7.3
select==python-3.7.3
selectors==python-3.7.3
setuptools==40.8.0
_sha1==python-3.7.3
_sha256==python-3.7.3
_sha3==python-3.7.3
_sha512==python-3.7.3
shelve==python-3.7.3
shlex==python-3.7.3
shutil==python-3.7.3
signal==python-3.7.3
_signal==python-3.7.3
site==python-3.7.3
_sitebuiltins==python-3.7.3
six==1.11.0
smtpd==Python SMTP proxy version 0.3
smtplib==python-3.7.3
sndhdr==python-3.7.3
socket==python-3.7.3
_socket==python-3.7.3
socketserver==0.4
spwd==python-3.7.3
sqlite3==2.6.0
_sqlite3==2.6.0
_sre==python-3.7.3
sre_compile==python-3.7.3
sre_constants==python-3.7.3
sre_parse==python-3.7.3
ssl==OpenSSL 1.0.2k-fips  26 Jan 2017
_ssl==OpenSSL 1.0.2k-fips  26 Jan 2017
stat==python-3.7.3
_stat==python-3.7.3
statistics==python-3.7.3
string==python-3.7.3
_string==python-3.7.3
stringprep==python-3.7.3
_strptime==python-3.7.3
struct==python-3.7.3
_struct==python-3.7.3
subprocess==python-3.7.3
sunau==python-3.7.3
symbol==python-3.7.3
symtable==python-3.7.3
_symtable==python-3.7.3
sys==python-3.7.3
sysconfig==3.7.3
_sysconfigdata_m_linux_x86_64-linux-gnu==python-3.7.3
syslog==python-3.7.3
tabnanny==6
tarfile==0.9.0
telnetlib==python-3.7.3
tempfile==python-3.7.3
termios==python-3.7.3
test==python-3.7.3
test_bootstrap==python-3.7.3
test_lambda_runtime_client==python-3.7.3
test_lambda_runtime_marshaller==python-3.7.3
_testbuffer==python-3.7.3
_testcapi==python-3.7.3
_testimportmultiple==python-3.7.3
_testmultiphase==python-3.7.3
textwrap==python-3.7.3
this==python-3.7.3
_thread==python-3.7.3
threading==python-3.7.3
_threading_local==python-3.7.3
time==python-3.7.3
timeit==python-3.7.3
tkinter==error:module
token==python-3.7.3
tokenize==python-3.7.3
trace==python-3.7.3
traceback==python-3.7.3
tracemalloc==python-3.7.3
_tracemalloc==python-3.7.3
tty==python-3.7.3
turtle==error:module
turtledemo==python-3.7.3
types==python-3.7.3
typing==python-3.7.3
unicodedata==11.0.0
unittest==python-3.7.3
urllib3==1.24.1
urllib==python-3.7.3
uu==python-3.7.3
uuid==python-3.7.3
venv==python-3.7.3
warnings==python-3.7.3
_warnings==python-3.7.3
wave==python-3.7.3
weakref==python-3.7.3
_weakref==python-3.7.3
_weakrefset==python-3.7.3
webbrowser==python-3.7.3
wsgiref==python-3.7.3
xdrlib==python-3.7.3
xml==python-3.7.3
xmlrpc==python-3.7.3
xxlimited==python-3.7.3
xxsubtype==python-3.7.3
_xxtestfuzz==python-3.7.3
zipapp==python-3.7.3
zipfile==python-3.7.3
zipimport==python-3.7.3
zlib==1.0