Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Compares list of imports within a Python codebase with its pip-requires file
#!/usr/bin/perl
use strict;
use warnings;
use File::Find;
@ARGV or unshift @ARGV, '.';
my %imports = ();
my %std_libs = get_std_libs();
find(\&detect_imports, @ARGV);
my $pip_requires_file = find_pip_requires();
my %pip_requires = get_pip_requires($pip_requires_file);
report(\%imports, \%pip_requires);
sub get_std_libs {
# Yes, I should have written this whole hack in Python :-(
my %std_libs;
my $python = <<'EOPYTHON';
# http://stackoverflow.com/a/8992937/179332
import distutils.sysconfig as sysconfig
import os
import sys
std_lib = sysconfig.get_python_lib(standard_lib=True)
for top, dirs, files in os.walk(std_lib):
for nm in files:
prefix = top[len(std_lib)+1:]
if prefix[:13] == 'site-packages':
continue
if nm == '__init__.py':
print top[len(std_lib)+1:].replace(os.path.sep,'.')
elif nm[-3:] == '.py':
print os.path.join(prefix, nm)[:-3].replace(os.path.sep,'.')
elif nm[-3:] == '.so' and top[-11:] == 'lib-dynload':
print nm[0:-3]
for builtin in sys.builtin_module_names:
print builtin
EOPYTHON
open(PYTHON, qq{python -c "$python" |})
or die "Couldn't run python: $!\n";
while (<PYTHON>) {
chomp;
$std_libs{$_}++;
#print "stdlib: [$_]\n";
}
close(PYTHON) or die "close(python|) failed: $!\n";
return %std_libs;
}
sub detect_imports {
return unless /\.py$/;
search_file($File::Find::dir, $_);
}
sub search_file {
my ($dir, $file) = @_;
open(FILE, $file)
or die qq{open(FILE, $file) failed: $!\n};
while (<FILE>) {
s/^\s+//;
s/#.*//;
s/\s+$//;
if (/^import\s+(.+?)(\s+as\s+(\w\S+))?$/) {
my $imports = $1;
for (split /\s*,\s*/, $imports) {
$imports{$_}++ unless $std_libs{$_};
# print "Detected import: $_\n" unless $std_libs{$_};
# print "Ignoring standard library import: $_\n" if $std_libs{$_};
}
}
elsif (/^from\s+(.+)\s+import\s+/) {
$imports{$1}++ unless $std_libs{$1};
# print "Detected from import: $_\n" unless $std_libs{$_};
# print "Ignoring standard library import: $_\n" if $std_libs{$_};
}
}
close(FILE);
}
sub find_pip_requires {
chomp(my $lazy_hack = `find -name pip-requires`);
die "Found more than one pip-requires?!\n$lazy_hack\n"
if $lazy_hack =~ /\n/;
return $lazy_hack;
}
sub get_pip_requires {
my %pip_requires = ();
my ($pip_requires_file) = @_;
open(PIP, $pip_requires_file)
or die qq{open(PIP, $pip_requires_file) failed: $!\n};
while (<PIP>) {
chomp;
s/[<=>]=.+//;
$pip_requires{$_}++;
}
close(PIP);
return %pip_requires;
}
sub header {
my ($header) = @_;
print "$header\n", "=" x length($header), "\n\n";
}
sub report {
my ($imports, $pip_requires) = @_;
header("packages imported and in pip-requires");
my %union = (%$imports, %$pip_requires);
for my $package (sort keys %union) {
print "$package\n" if $imports->{$package} && $pip_requires->{$package};
}
print "\n";
header("packages imported but not in pip-requires");
for my $package (sort keys %$imports) {
print "$package\n" unless $pip_requires->{$package};
}
print "\n";
header("packages in pip-requires but not imported");
for my $package (sort keys %$pip_requires) {
print "$package\n" unless $imports->{$package};
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment