gradleの依存関係とキャッシュ上のpomファイルを照合して依存関係のjsonを出力するスクリプト

makeDepsJson.pl

#!/usr/bin/perl --

# - カレントディレクトリで./gradlew :app:dependencies して依存関係を列挙する
# - ユーザフォルダの.gradle/ にあるpomファイルを探索する
# - 依存関係とpomファイルを突き合わせて json を出力する

use 5.32.1;
use strict;
use warnings;
use Getopt::Long;
use File::Find;
use File::Path qw(make_path);
use File::Copy;
use JSON::XS;
use Types::Serialiser;
use constant{
    true =>Types::Serialiser::true,
    false =>Types::Serialiser::false,
};
use XML::XPath;
use XML::XPath::XMLParser;
use Data::Dump qw(dump);

# オプション
my $gradleDir = '/c/Users/tateisu/.gradle';
my $outFile = "app/src/main/res/raw/dep_list.json";
GetOptions (
   "gradleDir=s" => \$gradleDir,
   "outFile=s" => \$outFile,
) or die("bad options.\n");

###############

# gradleフォルダ探索時に無視するディレクトリ
my %ignoreDirNames = map{ ($_,1) } qw(
    . .. .tmp kotlin-dsl
);

# 以下のライブラリはpomにDevelopers指定がなくても許容する
my @libsMissingDevelopers = qw(
    androidx.databinding:databinding-adapters
    androidx.databinding:databinding-
    androidx.databinding:viewbinding
    com.google.android.datatransport:transport-
    com.amazonaws:aws-android-sdk-
    com.google.code.findbugs:jsr305:
    com.google.android.gms:play-services-
    com.google.code.gson:gson:
    com.google.errorprone:error_prone_annotations:
    com.google.firebase:firebase-
    com.google.guava:failureaccess
    com.google.guava:guava
    com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
    com.jakewharton.picasso:picasso2-okhttp3-downloader
    com.squareup.picasso:picasso
    com.theartofdev.edmodo:android-image-cropper
    io.realm:android-adapters
    javax.inject:javax.inject
    org.apache.httpcomponents:httpclient
    org.apache.httpcomponents:httpcore
    org.apache.httpcomponents:httpmime
    org.eclipse.paho:org.eclipse.paho.client.mqttv3
);

# 以下のライブラリはpomにwebSite指定がなくても許容する
my @libsMissingWebSite = qw(
    androidx.databinding:databinding-adapters
    androidx.databinding:databinding-
    androidx.databinding:viewbinding
    com.google.android.datatransport:transport-
    com.google.android.gms:play-services-
    com.google.code.gson:gson:
    com.google.errorprone:error_prone_annotations:
    com.google.firebase:firebase-
    com.google.guava:failureaccess
    com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
    com.jakewharton.picasso:picasso2-okhttp3-downloader
    com.squareup.picasso:picasso
    com.theartofdev.edmodo:android-image-cropper
    org.eclipse.paho:org.eclipse.paho.client.mqttv3
);

# 以下のライブラリはpomにライセンス指定がなくても許容する
my @libsMissingLicenses = qw(
    com.google.guava:failureaccess
    com.google.guava:guava
    com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
    com.squareup.picasso:picasso
    com.theartofdev.edmodo:android-image-cropper
    commons-codec:commons-codec
    commons-logging:commons-logging
    org.apache.httpcomponents:httpclient
    org.apache.httpcomponents:httpcore
    org.apache.httpcomponents:httpmime
    org.eclipse.paho:org.eclipse.paho.client.mqttv3
);

# 以下のライブラリはpomにライセンス名の指定がなくても許容する
my @libsMissingLicenseName = qw(
    net.zetetic:android-database-sqlcipher
);

# idがprefixesリストのいずれかに前方一致するなら真
sub matchLibs($$){
    my($id,$prefixes)=@_;
    for my $prefix(@$prefixes){
        return true if $id =~/\A$prefix/;
    }
    return false;
}

########################################################
# gradle で依存関係を列挙する

say "listing dependencies..";
my %conf;
{
    my $lastConf;

    open(my $fh,"-|","./gradlew -q --no-configuration-cache :app:dependencies --configuration productionReleaseRuntimeClasspath")
    or die "failed to get dependencies: $!";
    while(<$fh>){
        s/\s+\z//;
        s/[\x0d\x0a]+//;
        my $origLine = $_;

        next if not length;
        next if $_ eq "No dependencies";
        next if $_ =~ /^Project/;
        next if /\A-+\z/;

        my $lv = 0;
        if( s/\A([ \\|+-]+)// ){
            $lv = int(length($1)/5);
            $lv > 0 or die "invalid indent: [$origLine]";

            next if /^project :/;
            s/\s*\Q(*)\E$//;
            s/\s*\Q(c)\E$//;
            
            # バージョンのみが変わる場合
            s/([^ :]+?) -> ([^ :]+?)$/$2/;
            # パッケージごと変わる場合
            s/(\S+?) -> (\S+?)$/$2/;

            $lastConf or die "missing lastConf.";
            $lastConf->{deps}{$_} = 1;
        }else{
            $lastConf = $conf{$_}={
                name => $_
                ,deps => {}
            };
        }
    }
    close($fh) or die "failed to get dependencies: $!";
}

########################################################
# gradleのキャッシュからPOMファイルを列挙する

say "reading pom from $gradleDir ...";
my %pom;
find({
    no_chdir =>true,
    ,preprocess=>sub{
        return grep{
            if(not -d "$File::Find::dir/$_" ){
                true
            }else{
                if( $ignoreDirNames{$_} || /^transforms/ ){
                    false
                }else{
                    true
                }
            }
        } @_;
    }
    ,wanted =>sub{
        my $fileName = $File::Find::name;
        if( $fileName =~ /\.pom$/ ){
            my $xp = XML::XPath->new(filename => $fileName);

            my $groupId = $xp->findvalue('/project/groupId')->value()
                 || $xp->findvalue('/project/parent/groupId')->value()
                 || die "missing groupId in $fileName";

            my $artifactId = $xp->findvalue('/project/artifactId')->value()
                 || $xp->findvalue('/project/parent/artifactId')->value()
                 || die "missing artifactId in $fileName";

            my $version = $xp->findvalue('/project/version')->value()
                || $xp->findvalue('/project/parent/version')->value()
                || die "missing version in $fileName";
            
            $pom{"$groupId:$artifactId:$version"} ={
                    pomFile => $fileName,
                    groupId => $groupId,
                    artifactId => $artifactId,
                    version => $version,
            } ;
        }
    }
},"$gradleDir/caches");

##########################################################
# merging dependencies and poms

my @founds;
my @missings;
for my $confName (sort keys %conf){
    my $conf = $conf{$confName};
    my @deps = sort keys %{ $conf->{deps}};
    next if not @deps;
    say $confName;
    for my $dep(@deps){
        my $pomInfo = $pom{$dep};

        if(not $pomInfo){
            $dep =~ s/com.squareup.okio:okio/com.squareup.okio:okio-jvm/;
            $dep =~ s/io.insert-koin:koin-core/io.insert-koin:koin-core-jvm/;
            $pomInfo = $pom{$dep};
        }

        $pomInfo or push @missings,$dep;
        $pomInfo and push @founds,{
            id => $dep,
            pomInfo=>$pomInfo,
        }
   }
}

if(@missings){
    # 解決できなかった。不足分とpomファイルの一覧を出して終了。
    for(@missings){
        say "missing pom for $_";
    }
    for(sort keys %pom){
        say "pom: $_->{id}";
    }
    exit 1;
}

my $pomDir = "depPoms";
make_path($pomDir);
my @info;
my @errors;
LOOP: for(@founds){
    my $pomInfo = $_->{pomInfo};
    my $id = $_->{id};

    # デバッグ用：pomファイルをコピーする
    # スクリプトから使う訳ではない
    {
        my $outPomFile = "$id.pom";
        $outPomFile =~ s/:/_/g;
        if(not -e "$pomDir/$outPomFile"){
            copy($_->{pomInfo}{pomFile}, "$pomDir/$outPomFile");
        }
    }

    my $xp = XML::XPath->new(filename => $_->{pomInfo}{pomFile});

    my $info = {
        id => $id,
    };
    push @info,$info;

    my $developers = $info->{developers} = [];
    for my $node( $xp->findnodes("/project/developers/developer") ){
        my $name = $node->findvalue("name")->value()
            || $node->findvalue("id")->value();
        if(not $name){
            push @errors,"[$id]missing developer.name";
        }else{
            push @$developers,{
                name => $name,
            };
        }
    }

    if( not @$developers 
    and not matchLibs($id,\@libsMissingDevelopers) 
    ){
        push @errors,"[$id]missing developers.";
    }

    my $licenses = $info->{licenses} = [];
    for my $node( $xp->findnodes("/project/licenses/license") ){
        my $name = $node->findvalue('name')->value();
        if( not $name and matchLibs($id,\@libsMissingLicenseName) ){
            $name = "Unknown license"
        }
        if( not $name){
            push @errors,"[$id]missing license.name";
            next;
        }

        my $url = $node->findvalue('url')->value()
            or die "[$id]missing license.url";
        
        push @$licenses, {
            name => $name,
            url => $url,
        };
    }
    if( not @$licenses
    and not matchLibs($id,\@libsMissingLicenses)
    ){
        push @errors,"[$id]missing licenses.";
    }

    my $name = $xp->findvalue('/project/name')->value();
    $name and $info->{name} = $name;

    my $description = $xp->findvalue('/project/description')->value();
    if($description){
        $description =~ s/\A\s+//;
        $description =~ s/\s+\z//;
        $description and $info->{description} = $description;
    }

    my $webSite = $info->{website} = $xp->findvalue('/project/url')->value()
        || $xp->findvalue('/project/scm/url')->value();

    if( $webSite){
         $info->{website} = $webSite;
    }else{
        if( not matchLibs($id,\@libsMissingWebSite) ){
            push @errors,"[$id]missing website.";
        }
    }

    $info->{artifactVersion} = $pomInfo->{version};
}

if( @errors){
    say $_ for @errors;
    exit 1;
}

## ライセンスを@licenses にまとめる
## あらかじめよくあるライセンスのURL(バリエーションがある…)を列挙することで、URLの細かい変化による重複を吸収する
my @licenses = (
    {
        name => "The Apache Software License, Version 2.0",
        shortName => "Apache-2.0",
        urls =>[
            "https://www.apache.org/licenses/LICENSE-2.0.txt",
            "https://www.apache.org/licenses/LICENSE-2.0",
            "http://www.apache.org/licenses/LICENSE-2.0.txt",
            "http://www.apache.org/licenses/LICENSE-2.0",
            "https://api.github.com/licenses/apache-2.0",
        ],
    },
    {
        name => "MIT License",
        shortName => "MIT",
        urls =>[
            "https://opensource.org/license/mit/",
            "https://github.com/lisawray/groupie/blob/master/LICENSE.md",
            "https://github.com/omadahealth/SwipyRefreshLayoutblob/master/LICENSE",
        ],
    },
    {
        name => "The 2-Clause BSD License",
        shortName => "BSD-2-Clause",
        urls =>[
            "https://opensource.org/license/bsd-2-clause/",
            "http://www.opensource.org/licenses/bsd-license",
        ],
    },
    {
        name =>"SQLCipher Community Edition License",
        shortName => "SQLCipher Community Edition License",
        urls =>[
            "https://www.zetetic.net/sqlcipher/license/",
        ],
    },
    {
        name => "Amazon Software License",
        shortName => "Amazon Software License",
        urls =>[
            "https://aws.amazon.com/asl/",
            "http://aws.amazon.com/asl/",
        ],
    },
    {
        name => "Unicode, Inc. License",
        shortName => "Unicode License",
        urls =>[
            "https://www.unicode.org/copyright.html#License",
            "http://www.unicode.org/copyright.html#License",
        ],
    },
);

sub findLisenceByUrl($){
    my($url) = @_;
    for( @licenses){
        return $_ if grep{ $_ eq $url } @{$_->{urls}};
    }
    return;
}

sub createLicenseShortName($){
    my($json)=@_;

    my($oldItem) = findLisenceByUrl($json->{url});
    $oldItem and return $oldItem->{shortName};

    my $shortName = $json->{name};
    push @licenses,{
        shortName => $shortName,
        name => $json->{name},
        urls =>[ $json->{url} ],
    };
    return $shortName;
}

for my $info (@info){
    @{$info->{licenses}} = map{ createLicenseShortName($_) } @{$info->{licenses}};
}

for(@licenses){
    my $url = $_->{urls}[0];
    say "$_->{shortName} $_->{name} $url";
}

open(my $fh,">",$outFile) or die "$outFile $!";
print $fh encode_json {
    libs=> \@info,
    licenses => \@licenses,
};
close($fh) or die "$outFile $!";