A simple script that can help you find any jobs that are stuck in the state BackoffLimitExceeded after failure. Note that this will depend on using jq-1.6 and GNU-Sed in this example.

findStuckJobs.sh

#!/bin/bash

# Fetch the list of jobs to evaluate...
JOBS_TO_EVALUATE=$(
  kubectl get job -o json  \
   | jq '.items[] | .metadata.name' \
   | sed 's/"//g' \
  );

# Find jobs that are stuck in the state BackoffLimitExceeded
for job in $JOBS_TO_EVALUATE; do
  if [[ $(kubectl get job $job -o jsonpath='{.status.conditions[?(@.reason=="BackoffLimitExceeded")].status}') == *True* ]]; then
    echo "WARNING - job $job is in state BackoffLimitExceeded"
  fi
done