Find Dockerfile's recursively and report on possible upgrades.

demo.console

./find-latest-images.sh
Found configuration /home/ah/.config/find-latest-images/find-latest-images.yaml
Processing Dockerfile ./Dockerfile
Processing image docker.io/fluent/fluentd-kubernetes-daemonset
Hit /home/ah/.cache/find-latest-images/docker.io/fluent/fluentd-kubernetes-daemonset-tags.json
Processing image docker.io/postgres
Hit /home/ah/.cache/find-latest-images/docker.io/postgres-tags.json
Processing image foo
Querying tags of foo
time="2024-05-31T22:23:04+02:00" level=fatal msg="Error listing repository tags: fetching tags list: requested access to the resource is denied"
Processing image envoyproxy/envoy
Hit /home/ah/.cache/find-latest-images/envoyproxy/envoy-tags.json



Dockerfile  Image                                  Current                                  New
Dockerfile  d/fluent/fluentd-kubernetes-daemonset  v1.16.5-debian-elasticsearch7-amd64-1.0  UNCHANGED
Dockerfile  d/postgres                             15.2                                     15.7
Dockerfile  foo                                    15.2                                     NOT_FOUND
Dockerfile  envoyproxy/envoy                       v1                                       v1.30.1

output of /usr/bin/time:
0.09user 0.15system 0:01.67elapsed 14%CPU (0avgtext+0avgdata 24192maxresident)k
0inputs+16outputs (1major+52317minor)pagefaults 0swaps

Dockerfile

FROM docker.io/fluent/fluentd-kubernetes-daemonset:v1.16.5-debian-elasticsearch7-amd64-1.0
FROM docker.io/postgres:15.2
FROM foo:15.2
FROM envoyproxy/envoy:v1

find-latest-images.sh

#!/bin/bash
set -e

declare -a missing_deps
for dep in jq yq skopeo; do
    if ! command -v $dep >/dev/null; then
        missing_deps+=($dep)
    fi
done
if [ "${missing_deps[@]}" ]; then
    echo "Please install $missing_deps"
    exit 1
fi

CACHE=~/.cache/find-latest-images
CACHE_TTL_MIN=60
CONFIG=~/.config/find-latest-images/find-latest-images.yaml

report=$(mktemp)
trap "rm -f $report" EXIT ERR SIGINT

if [ -s "$CONFIG" ]; then
  echo "Found configuration $CONFIG"
fi

# returns 0 if Dockerfile should be processed, returns 1 otherwise and echo the reason.
function filterDockerfile() {
    local dockerfile="$1"
    if ! [ -s "$CONFIG" ]; then
        return 0
    fi
    while read regex; do
        if [[ $dockerfile =~ $regex ]]; then
            echo $(yq ".dockerfiles.skip[] | select(.regex == \"$regex\") | .reason // .regex" "$CONFIG")
            return 1
        fi
    done < <(yq .dockerfiles.skip[].regex "$CONFIG")
}

# returns 0 if image should be processed, returns 1 otherwise and echo the reason.
function filterImage() {
    local image="$1"
    if ! [ -s "$CONFIG" ]; then
        return 0
    fi
    while read regex; do
        if [[ $image =~ $regex ]]; then
            echo $(yq ".images.skip[] | select(.regex == \"$regex\") | .reason // .regex" "$CONFIG")
            return 1
        fi
    done < <(yq .images.skip[].regex "$CONFIG")
}

# Query the tags with skopeo or hit the cache
function listTags() {
    local image="$1"
    local cache="$CACHE/${image}-tags.json"

    # under 50 bytes, we can assume cache is garbage
    if [ "$(find "$cache" -mmin -$CACHE_TTL_MIN -size +50c 2>/dev/null)" ]; then
        echo "Hit $cache" >&2
    else
        mkdir -p $(dirname "$cache")
        echo "Querying tags of $image" >&2
        skopeo list-tags "docker://$image" > "$cache"
    fi
    jq -r .Tags[] "$cache"
}

# Filter the tags, expecting to receive one by line of stdin
function filterTags() {
    local image="$1"
    local current="$2"

    if [ -z "$current" ] || [ "$current" == latest ]; then
        echo "$current"
        return
    fi

    # Some docker repositories have tens of thousands of tags
    # It's super slow to filter through them
    # I'm using named pipe to streamline their processing
    # I'm programmatically creating a chain of command such as `cat tags | grep regex | grep regex | ...`
    # because the number of required `grep regex` depends on each image and isn't known in advance
    # The way it works is:
    #   1. create input named pipe
    #   2. create output named pipe
    #   3. set the grep to consume input and produce to output
    #   4. repeat as many times as you need grep
    #   5. feed start of the chain
    local pipeDir=$(mktemp -d)
    trap "rm -rf $pipeDir" RETURN

    declare -a pipes
    pipe=$pipeDir/${#pipes[@]}
    pipes+=($pipe)
    mkfifo $pipe

    # A freeze rule defines a regex to apply on the original tag.
    # The captured match is described as "frost".
    # Only accept candidate tag if it contains the "frost".
    # by default, freeze the major version, otherwise used configured rules
    # If the regex started with ^ or ended with $, prefix/postfix the frost before trying to match the candidate tag.
    if [ $(yq ".versions.\"$image\".freeze | length" "$CONFIG") -gt 0 ]; then
        while read freezeRegex; do
            # sed escapes literal dots
            local frost=$(grep -Po "$freezeRegex" <<< "$current" | sed 's/[.]/[.]/g')
            if [ -z "$frost" ]; then
                continue
            fi
            if [[ "$freezeRegex" == ^* ]]; then
                frost=^$frost
            fi
            if [[ "$freezeRegex" == *$ ]]; then
               frost=$frost\$
            fi

            pipe=$pipeDir/${#pipes[@]}
            pipes+=($pipe)
            mkfifo $pipe

            grep -P "$frost" < ${pipes[-2]} > ${pipes[-1]} &
        done < <(yq ".versions.\"$image\".freeze[]" "$CONFIG")
    else
        local current_major="${current%%.*}"

        pipe=$pipeDir/${#pipes[@]}
        pipes+=($pipe)
        mkfifo $pipe

        grep -P "^$current_major[.]" < ${pipes[-2]} > ${pipes[-1]} &
    fi

    if [ $(yq ".versions.\"$image\".accept | length" "$CONFIG") -gt 0 ]; then
        # Joining the accept rules as a regex group with multiple alternatives
        local accept='('$(yq '.versions.\"$image\".accept | join("|")' "$CONFIG" 2>/dev/null)')'
    else
        local accept='^v?[0-9]+([.][0-9]+([.][0-9]+)?)?$'
    fi

    grep -P "$accept" < ${pipes[-1]} &
    cat > ${pipes[0]}
}

# Format image for better presentation
function formatImage() {
    local image="$1"

    while read replace; do
        local by=$(yq ".images.report[] | select(.replace == \"$replace\") | .by" "$CONFIG")
        local image=$(sed "s|$replace|$by|" <<< "$image")
    done < <(yq .images.report[].replace "$CONFIG")

    echo $image
}

# Format Dockerfile for better presentation
function formatDockerfile() {
    local dockerfile="$1"

    if yq '.dockerfiles.report[].artifactId-instead' "$CONFIG" | grep -q true; then
        if [ -f "$(dirname "$dockerfile")/pom.xml" ]; then
           yq .project.artifactId "$(dirname "$dockerfile")/pom.xml"
           return
        fi
    fi

    realpath --relative-to=. "$dockerfile"
}

while read dockerfile; do
    if reason=$(filterDockerfile "$dockerfile"); then
        echo "Processing Dockerfile $dockerfile"
    else
        echo "Skipping Dockerfile $dockerfile because $reason"
        continue
    fi

    while read imageV; do
        image=$(cut -d: -f1 <<< "$imageV")
        current=$(cut -sd: -f2 <<< "$imageV")

        if reason=$(filterImage "$image"); then
            echo "Processing image $image"
        else
            echo "Skipping image $image because $reason"
            continue
        fi

        new_v=$(listTags "$image" | filterTags "$image" "$current" | sort -V | tail -1)

        if [ "$current" ] && [ -z "$new_v" ]; then
            v_status=NOT_FOUND
        elif [ "$current" == "$new_v" ]; then
            v_status=UNCHANGED
        else
            v_status="$new_v"
            # sed -i "s|$image:$current|$image:$new_v|" "$dockerfile"
        fi

        echo "$(formatDockerfile "$dockerfile")	$(formatImage "$image")	$current	$v_status" >> $report

        unset image current reason new_v v_status
    done < <(grep -Po "^FROM \K\S+" "$dockerfile")
done < <(find . -type f -name Dockerfile -not -path "./.git/*" -not -path "*/src/it/*" -not -path "*/src/test/*" -not -path "*/target/*")

echo -e '\n\n'
column -t -s$'\t' -N Dockerfile,Image,Current,New $report

find-latest-images.yaml

---
dockerfiles:
  report:
    - artifactId-instead: true
  skip:
    - regex: .*/archetype-resources/.*
      reason: it is an archetype and not a real dockerfile
    - regex: .*/helm-unit-tests/.*
      reason: it is used by a test tool and uses a symbolic version anyway
images:
  report:
    - replace: docker.io
      by: d
    - replace: quay.io
      by: q
  skip:
    - regex: "fita.dev/.*"
      reason: "it is not a dependency"
versions:
  docker.io/fluent/fluentd-kubernetes-daemonset:
    freeze:
      - debian-elasticsearch7
      - debian-kafka2
      - ^v[0-9]+
    accept:
      - debian-elasticsearch7(-amd64)?
      - debian-kafka2(-amd64)?
      - ^v[0-9]+([.][0-9]+([.][0-9]+)?)?-.*