checking for useless requirements in requirements.txt in python

check_require.sh

#!/usr/bin/env bash
# author julien@tayon.net
# licence WTFPL2.0
WORKDIR=./__
WVENVDIR=todel
GRUIK=
PYTHON=
export WVENVDIR
UPGRADE_PIP=hygien
TO_KEEP_OR_NOT_TO_KEEP="DONT_KEEP"
AUDIT=""
RZ="\e[0m"
GD="\e[32m"
RD="\e[31m"
BL=$( tput bold )
function abs_filename {
  # $1 : relative filename
  echo "$(cd "$(dirname "$1")" && pwd)/$(basename "$1")"
}
function cleanup {
    if [ ! ${GRUIK} ]; then
        [ -d "${WORKDIR}" ] && rm "${WORKDIR}" -rf
        [ -d "${WVENVDIR}" ] && rm "$WVENVDIR" -rf 
        >&2 echo -e "clean up ${GD}OK${RZ}"
    else
        >&2 echo -e "${BL}${RD}No mr proper for you: gruik gruik BUT ${GD}Caches are HOT${RZ}"
     fi
}

function usage {
    cat <<EOF
        $( basename $0 ) [-g] [-k] [-s] [-p path/to/python] [-s] [-d dumpfile] path/to/requirements.txt

Check dependencies in a python requirements are not over-specified

        -k shows the line to keep, else (default) shows which dependencies
           already requires this dependency

        -g gruik mode : keep cache and live with the consequences

        -s dont check for CVE (as reported in pypi)

        -a print an audit of the package

        -d dump a list of modules 
           effectively installed in the virtualenv (requires -a)

        -p python version used for testing the requirements

        -t don't update pip pliz.


        requires vi jq and curl to work
EOF
    exit 85;
};

# on exit with ctrl+c cleanup the working dirs 
# to not keep states that could create false positive
trap cleanup EXIT SIGINT SIGTERM
DUMP=""
CHECK_CVE="pas_faux"
while getopts ":hgksap:d:t" Option
do
    case "${Option}" in
        k  ) TO_KEEP_OR_NOT_TO_KEEP="KEEP";;
        a  ) AUDIT=1;;
        g  ) >&2 GRUIK="gruik" \
                && echo -e "GRUIK MODE ON: ${RD}possible false positive ${RZ} and ${GD}hot cache${RZ}";;
        d ) DUMP=${OPTARG};;
        t ) UPGRADE_PIP="";echo "tainted love";;
        s ) CHECK_CVE="";;
        p ) PYTHON=$( echo "-p \"${OPTARG}\"" );;
        h|* ) usage ;;   # Default.
    esac
done

shift $(($OPTIND - 1))
[[ ! "$GRUIK" ]] && cleanup
[ ! -d $WORKDIR ] && mkdir $WORKDIR
[ -z "$2" ] || ( echo "too much argument" ; usage ;)
[ "$1" ] && { [ ! -f "$1" ] && { echo -e "\n$1::requirements not found\n"; usage; } }
# remove bom the dirty way
echo -n "converting $1 in utf8 no BOM unix EOL "
>&2 vi -c ":set nobomb fileencoding=utf8 ff=unix" -c ":wq" "$1" &> /dev/null && echo -e ${GD}OK$RZ || echo -e "${RD}KO$RZ"

INPUT="${1}"
function get_json {
    module=${1,,}
    if [ ! -f "${WORKDIR}/${module}.json" ]; then
        curl -s https://pypi.org/pypi/${module}/json > ${WORKDIR}/${module}.json
    fi
}
function check_vuln {
    module=${1,,}
    get_json $module
        jq '.vulnerabilities | select(length>0)' $WORKDIR/${module}.json 
}

function get_module {
    perl -ane '/^([a-z0-9A-Z\-\._]+)/ and print "${1,,}\n"';
}
function get_version {
    get_json ${1,,}
    echo $( jq .info.version <  $WORKDIR/${1,,}.json )
}
for i in $( cat $INPUT | perl -ane '/^([a-z0-9A-Z\-\._]+)/ and print "$1\n"' ); do
    i=${i,,}
    get_json $i


    cat $WORKDIR/$i.json \
        | jq .info.requires_dist  \
        | perl -ane '/"([^"]+)"/ and print "$1\n"' > $WORKDIR/$i.dep ;
    cat $WORKDIR/$i.json \
        | jq .info.install_requires  \
        | perl -ane '/"([^"]+)"/ and print "$1\n"' >> $WORKDIR/$i.dep ;

done

cat "$INPUT" | while read -r l; do
module=$(echo $l | perl -ane '/^([a-zA-Z0-9\-\._]+)/ and print "$1\n"');
module=${module,,}
    if [[ "$module" ]]; then
        useless=$(grep -iP "^(${module})(\\s|\\$)" $WORKDIR/*.dep | grep -v extra)
       if [[ $CHECK_CVE ]]; then
           check_vuln $module
        fi
        if [[ $TO_KEEP_OR_NOT_TO_KEEP == "KEEP" ]] && [ -z "$useless" ]; then
           echo $l
        fi
        if [[ $TO_KEEP_OR_NOT_TO_KEEP != "KEEP" ]] && [ "$useless" ]; then
           echo
           echo $l
           echo "required by"
           echo $useless
           echo "-----------------------------------------------------"
       fi
    fi
done
if [[ $AUDIT ]]; then
    echo
    echo checking requirements.txt is valid
    echo "**********************************"
    function onsubcfail {
        >&2 echo -e "$1 ${RD}ko${RZ}"
        exit 126;
    }
    virtualenv $PYTHON $WVENVDIR 1> /dev/null \
        || onsubcfail "virtual env setup"

    if [ -f "./${WVENVDIR}/Scripts/activate" ]; then
        ACTIVATE=./${WVENVDIR}/Scripts/activate
    else
        ACTIVATE=./${WVENVDIR}/bin/activate
    fi
    . $ACTIVATE

    [[ "$UPGRADE_PIP" ]] \
        && ( >&2 echo -n "upgrading pip " && python -mpip install --upgrade "pip!=22.2.2" 1> /dev/null && echo -e ${GD}OK$RZ || echo -e ${RD}KO${RZ} )

    echo -e "${BL}pip install -r $INPUT${RZ}"
    echo -e "you can follow it with$BL tail -f $( abs_filename ${WORKDIR}/res.txt )$RZ"
    ( python -mpip install -r $INPUT  1> ${WORKDIR}/res.txt \
        && echo -e "requirements install ${GD}ok${RZ}"  \
        || onsubcfail "requirements intall" ) &
    PID=$!
    p=( \| / - \\ )
    i=0
    sleep 1;
    while kill -0 $PID &> /dev/null; do
        echo -ne "${p[$(( $i % 4 ))]} $i seconds\\r"
        i=$(( $i +1 ))
        sleep 1
    done
    echo install took $i seconds

    for i in $( python -mpip list --format=freeze \
        | perl -ane '/^([a-z0-9A-Z\-\._]+)/ and print "$1\n"' ); do

        i=${i,,}
        get_json $i
        cat $WORKDIR/$i.json \
            | jq .info.requires_dist  | \
            perl -ane '/"([^"]+)"/ and print "$1\n"' > $WORKDIR/$i.dep ;
    done
    echo
    echo OUTDATED
    echo "********"
    echo 
    python -mpip list -o --format=freeze | while read -r l; do
        module=$(echo $l | get_module )
        version="$(get_version $module)"
        echo $l
        if [[ $version ]]; then 
            echo "   $module current is $version"
        fi
    done
    echo
    echo UP TO DATE
    echo "**********"
    echo
    python -mpip list -u --format=freeze
    echo
    if [[ $CHECK_CVE ]]; then 
        echo Checing vulnerabilities
        echo "***********************"
        echo
        for module in $( python -mpip list --format=freeze | get_module ); do
            vuln=$(check_vuln $module )
            if [[ $vuln ]]; then
            echo -e "$module ${RD}ko$RZ"
                echo ------------------
                echo
                echo $vuln
                echo
            fi
        done
    fi
    echo
    echo rechecking shortest dependency
    echo "******************************"
    echo
    >&2 echo -n "installing pipdeptree & graphviz " && python -mpip install pipdeptree graphviz  1> /dev/null && echo -e ${GD}OK$RZ || echo -e ${RD}KO${RZ}
    echo
    echo most roots packages
    echo
    pipdeptree -e "pipdeptree,pip,pkg-resources,graphviz"  -l -a --json-tree | jq -M -r '.[] | .key + "==" + .installed_version' > shortest.requirements.txt
    echo
    echo -e "shortest requirements list dumped in ${BL}$( abs_filename shortest.requirements.txt)${RZ}"
    echo
    cat shortest.requirements.txt
    echo
    echo -en "dumping a dependency graph in ${BL} $( abs_filename this.dot)$RZ "
    pipdeptree -l -a  --graph-output dot > this.dot && echo -e ${GD}OK${RZ} || echo -e ${RD}OK${RZ}
    vim -c "$( for i in $( cat shortest.requirements.txt | perl -ane '/^([^=]+)/;$s=$1; $s =~ s/(\.|\-)/\\$1/g; print "$s\n"' ) ; do printf '1,$s/\(%s"\\= \[.*\)\]/\\1 shape=box style=filled fillcolor=yellow]/i |' $i; done ) wq"  this.dot &> /dev/null



    echo
    if [[ "$DUMP" ]]; then
        echo Dumping list of installed packages
        echo "**********************************"
        echo
        python -m pip list --format=freeze > $DUMP \
            &&  echo "list of installed packages dumped in $DUMP"
        echo
    fi
    echo deleting virtualenv
    echo "*******************"
    echo
    deactivate
     if ( which dot ) &> /dev/null; then
        echo "graph dumped in $( abs_filename this.png )"
        dot this.dot -Tpng > this.png
    fi
    if ( which xdot ) &> /dev/null; then
        xdot this.dot
    fi
   
fi

nice 👍

@matrixise j'ai rajouté une partie audit (outdated/vuln/uptodate) et je tente la portabilité linux/window (git bash)