Skip to content

Instantly share code, notes, and snippets.

@Xiard
Forked from lrq3000/pylistmodules.py
Last active April 29, 2019 21:01
Show Gist options
  • Save Xiard/f9eb659776d806b52bf1fcdca7628300 to your computer and use it in GitHub Desktop.
Save Xiard/f9eb659776d806b52bf1fcdca7628300 to your computer and use it in GitHub Desktop.
List recursively all imports of modules along with versions done from your Python application.Tested on Python 2.7. No dependencies except standard Python libs.
from __future__ import print_function
# !/usr/bin/env python
# encoding: utf-8
# Copyright (C) 2001-2007 Martin Blais. All Rights Reserved
# Copyright (C) 2010 Bear http://code-bear.com/bearlog/
# Copyright (C) 2013 lrq3000
# Excerpt from SnakeFood to recursively list all imports of modules using AST parsing
# Additions to print the versions of each module if available
import os, sys
import compiler
from compiler.ast import Discard, Const
from compiler.visitor import ASTVisitor
import numbers
def pyfiles( startPath, excludeFolders ) :
r = [ ]
d = os.path.abspath( startPath )
if os.path.exists( d ) and os.path.isdir( d ) :
for root, dirs, files in os.walk( d ) :
if is_root_excluded( root, excludeFolders ): continue
print('handle root: ' + root)
for f in files :
n, ext = os.path.splitext( f )
if ext == '.py' :
r.append( [ root, f ] )
return r
class ImportVisitor( object ) :
def __init__( self ) :
self.modules = [ ]
self.recent = [ ]
def visitImport( self, node ) :
self.accept_imports()
self.recent.extend( (x[ 0 ], None, x[ 1 ] or x[ 0 ], node.lineno, 0)
for x in node.names )
def visitFrom( self, node ) :
self.accept_imports()
modname = node.modname
if modname == '__future__' :
return # Ignore these.
for name, as_ in node.names :
if name == '*' :
# We really don't know...
mod = (modname, None, None, node.lineno, node.level)
else :
mod = (modname, name, as_ or name, node.lineno, node.level)
self.recent.append( mod )
def default( self, node ) :
pragma = None
if self.recent :
if isinstance( node, Discard ) :
children = node.getChildren()
if len( children ) == 1 and isinstance( children[ 0 ], Const ) :
const_node = children[ 0 ]
pragma = const_node.value
self.accept_imports( pragma )
def accept_imports( self, pragma = None ) :
self.modules.extend( (m, r, l, n, lvl, pragma)
for (m, r, l, n, lvl) in self.recent )
self.recent = [ ]
def finalize( self ) :
self.accept_imports()
return self.modules
class ImportWalker( ASTVisitor ) :
def __init__( self, visitor ) :
ASTVisitor.__init__( self )
self._visitor = visitor
def default( self, node, *args ) :
self._visitor.default( node )
ASTVisitor.default( self, node, *args )
def parse_python_source( fn ) :
contents = open( fn, 'rU' ).read()
ast = compiler.parse( contents )
vis = ImportVisitor()
compiler.walk( ast, vis, ImportWalker( vis ) )
return vis.finalize()
def find_imports_and_print( startPath, excludeFolders ) :
for d, f in pyfiles( startPath, excludeFolders ) :
print( d, f )
print( parse_python_source( os.path.join( d, f ) ) )
def is_root_excluded( root, excludeFolders ) :
return any( [ f for f in excludeFolders if root.startswith( f ) ] )
def find_imports( startPath, excludeFolders ) :
moduleslist = { }
# Get the list of .py files and iterate over
count = 0
fs = pyfiles( startPath, excludeFolders )
skipped = [ ]
print( 'procssing {} files'.format( len( fs ) ) )
for d, f in fs :
# For each .py file, parse and get the list of imports
full = os.path.join( d, f )
try :
mod = parse_python_source( full )
# For each imported module, store only the root module (eg: sys.os -> will store only sys)
for m in mod :
moduleslist[ m[ 0 ].split( "." )[ 0 ] ] = True
count += 1
print( '.', end = '' )
if count % 50 == 0 :
print( str( count ) )
except SyntaxError as e :
skipped.append( full )
# Return the list of unique modules names
if skipped :
print( '\nskipped files with errors:' )
for s in skipped : print( s )
print( '\nfind_imports done' )
return moduleslist.keys()
def import_module( module_name ) :
''' Reliable import, courtesy of Armin Ronacher '''
try :
__import__( module_name )
except ImportError :
exc_type, exc_value, tb_root = sys.exc_info()
tb = tb_root
while tb is not None :
if tb.tb_frame.f_globals.get( '__name__' ) == module_name :
raise exc_type, exc_value, tb_root
tb = tb.tb_next
return None
return sys.modules[ module_name ]
def find_versions( moduleslist ) :
''' Find the version of each module if available (and only for modules installed, does not work with locally included files) '''
modver = { }
# For each module
for mod in moduleslist :
ver = 'NA'
m = import_module( mod ) # Import the module
verlist = [ ]
if m is None : # The module is not installed
ver = 'Not installed'
# Else the module is installed and imported, we try to find the version
else :
# Iterate over all keys and try to find the version
for k, v in m.__dict__.iteritems() :
if ('version' in k.lower() or '__version__' in k.lower() or 'ver' in k.lower()) \
and isinstance( v, (basestring, numbers.Number) ) :
verlist.append( v )
# Store the version
if len( verlist ) > 1 :
modver[ mod ] = verlist
elif len( verlist ) == 1 :
modver[ mod ] = verlist[ 0 ]
else :
modver[ mod ] = ver
# Return a dict where the keys are the modules names and values are the versions
return modver
def scan_folder(root, excludeFolders):
import pprint
import os
moduleslist = find_imports( root, [os.path.join(root, f) for f in excludeFolders.split( '|' )] )
modver = find_versions( moduleslist )
print( 'List of modules imported:' )
print( moduleslist )
print( '-' * 50 )
print( 'List of modules and versions:' )
pprint.pprint( modver )
def scan_idtdf():
scan_folder(r'c:\deviceNet\devices\clockMatrix\tc\idtdf', r'tests|.idea|.git|camron')
def scan_clock_matrix():
scan_folder(r'c:\deviceNet\devices\clockMatrix\tc', r'testsPersonality|auto|camron|clock_matrix_driver|import|idtdf|.idea|.git')
def scan_vc3s():
scan_folder(r'c:\deviceNet\devices\vc3simc\tc', r'testsPersonality|auto|configs|testcode|toolkit|idtdf|.idea|.git')
if __name__ == '__main__' :
#scan_clock_matrix()
scan_vc3s()
#scan_idtdf()
@Xiard
Copy link
Author

Xiard commented Apr 29, 2019

Made some minor changes to support excluding certain folders.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment