Obtain a transitive closure for a list of Maven dependencies (including release dates)

README.md

Input format example (the date field is not mandatory):

{'groupId': 'org.sonatype.nexus.plugins', 'artifactId': 'nexus-ruby-plugin', 'version': '2.11.4-01', 'date': 1436480633}
{'groupId': 'org.apache.maven.archiva', 'artifactId': 'archiva-site', 'version': '1.0-beta-1', 'date': 1186902008}
{'groupId': 'com.yahoo.vespa', 'artifactId': 'linguistics', 'version': '6.158.42', 'date': 1508227582}
{'groupId': 'org.xwiki.commons', 'artifactId': 'xwiki-commons-repository-api', 'version': '8.0', 'date': 1458055140}

To run:

./dep-resolver.sh # expects to find file input.json
./release-date-resolver.sh |tee transitive.json

dep-resolver.sh

#!/usr/bin/env bash
# (c) 2021 - Georgios Gousios
# Public domain

# kafkacat -C -t fasten.mvn.full.rnd -b samos >input.json
cat input.json |
while read line ; do
    line=$(echo "$line" | tr  "'" '"')
    echo "$line"
    group=$(echo "$line"|jq '.groupId' | tr -d '"')
    artifact=$(echo "$line"|jq '.artifactId' | tr -d '"')
    version=$(echo "$line"|jq '.version' | tr -d '"')

    cat pom-tmpl.xml |
    sed -e "s/GROUP/$group/" |
    sed -e "s/ARTIFACT/$artifact/" |
    sed -e "s/VERSION/$version/" > pom.xml

    deps=$(mvn dependency:tree -DoutputType=tgf 2>&1| \
    grep -v WARN |
    grep -Po "[0-9]+ (([A-Za-z0-9.\-_])*:){1,6}"| \
    cut -f2 -d' '| \
    cut -f1,2,4 -d ':' | \
    sort | uniq | \
    grep -v 'depresolver') 

    echo $deps |tr ' ' '\n'
    echo $deps |tr ' ' '\n' >> deps.txt
done

release-date-resolver.sh

#!/usr/bin/env bash
# (c) 2021 - Georgios Gousios
# Public domain

cat deps.txt |
sort |
uniq |
while read dep ; do

    group=$(echo "$dep"| cut -f1 -d':')
    artifact=$(echo "$dep"| cut -f2 -d':')
    version=$(echo "$dep"| cut -f3 -d':')

    url="https://repo1.maven.org/maven2/$(echo "$group"|tr "." "/")/$artifact/$version/ "
    pom=$(curl -s "$url" | grep "href=\".*.pom\"" | tr -s ' ')

    hasTitle=$(echo "$pom" | grep title)

    if [ -z "$hasTitle" ]; then
        d=$(echo "$pom" | cut -f3,4 -d' ')
    else
        d=$(echo "$pom" | cut -f4,5 -d' ')
    fi

    if [ -z "$d" ]; then
        echo "$url" not found
    fi

    echo "{\"groupId\": \"$group\", \"artifactId\": \"$artifact\", \"version\": \"$version\", \"date\": $(date --date="$d" +"%s")}"
done