Compares list of imports within a Python codebase with its pip-requires file

analyse-py-module-deps.pl

#!/usr/bin/perl

use strict;
use warnings;

use File::Find;

@ARGV or unshift @ARGV, '.';

my %imports = ();

my %std_libs = get_std_libs();
find(\&detect_imports, @ARGV);
my $pip_requires_file = find_pip_requires();
my %pip_requires = get_pip_requires($pip_requires_file);
report(\%imports, \%pip_requires);

sub get_std_libs {
    # Yes, I should have written this whole hack in Python :-(
    my %std_libs;
    my $python = <<'EOPYTHON';
# http://stackoverflow.com/a/8992937/179332
import distutils.sysconfig as sysconfig
import os
import sys

std_lib = sysconfig.get_python_lib(standard_lib=True)

for top, dirs, files in os.walk(std_lib):
    for nm in files:
        prefix = top[len(std_lib)+1:]
        if prefix[:13] == 'site-packages':
            continue
        if nm == '__init__.py':
            print top[len(std_lib)+1:].replace(os.path.sep,'.')
        elif nm[-3:] == '.py':
            print os.path.join(prefix, nm)[:-3].replace(os.path.sep,'.')
        elif nm[-3:] == '.so' and top[-11:] == 'lib-dynload':
            print nm[0:-3]

for builtin in sys.builtin_module_names:
    print builtin
EOPYTHON
    open(PYTHON, qq{python -c "$python" |})
        or die "Couldn't run python: $!\n";
    while (<PYTHON>) {
        chomp;
        $std_libs{$_}++;
        #print "stdlib: [$_]\n";
    }
    close(PYTHON) or die "close(python|) failed: $!\n";
    return %std_libs;
}

sub detect_imports {
    return unless /\.py$/;

    search_file($File::Find::dir, $_);
}

sub search_file {
    my ($dir, $file) = @_;
    open(FILE, $file)
    or die qq{open(FILE, $file) failed: $!\n};
    while (<FILE>) {
        s/^\s+//;
        s/#.*//;
        s/\s+$//;
        if (/^import\s+(.+?)(\s+as\s+(\w\S+))?$/) {
            my $imports = $1;
            for (split /\s*,\s*/, $imports) {
                $imports{$_}++ unless $std_libs{$_};
                # print "Detected import: $_\n" unless $std_libs{$_};
                # print "Ignoring standard library import: $_\n" if $std_libs{$_};
            }
        }
        elsif (/^from\s+(.+)\s+import\s+/) {
            $imports{$1}++ unless $std_libs{$1};
            # print "Detected from import: $_\n" unless $std_libs{$_};
            # print "Ignoring standard library import: $_\n" if $std_libs{$_};
        }
    }
    close(FILE);
}

sub find_pip_requires {
    chomp(my $lazy_hack = `find -name pip-requires`);
    die "Found more than one pip-requires?!\n$lazy_hack\n"
        if $lazy_hack =~ /\n/;
    return $lazy_hack;
}

sub get_pip_requires {
    my %pip_requires = ();
    my ($pip_requires_file) = @_;
    open(PIP, $pip_requires_file)
        or die qq{open(PIP, $pip_requires_file) failed: $!\n};
    while (<PIP>) {
        chomp;
        s/[<=>]=.+//;
        $pip_requires{$_}++;
    }
    close(PIP);
    return %pip_requires;
}

sub header {
    my ($header) = @_;
    print "$header\n", "=" x length($header), "\n\n";
}

sub report {
    my ($imports, $pip_requires) = @_;

    header("packages imported and in pip-requires");
    my %union = (%$imports, %$pip_requires);
    for my $package (sort keys %union) {
        print "$package\n" if $imports->{$package} && $pip_requires->{$package};
    }
    print "\n";

    header("packages imported but not in pip-requires");
    for my $package (sort keys %$imports) {
        print "$package\n" unless $pip_requires->{$package};
    }
    print "\n";

    header("packages in pip-requires but not imported");
    for my $package (sort keys %$pip_requires) {
        print "$package\n" unless $imports->{$package};
    }
}