Skip to content

Instantly share code, notes, and snippets.

@v1v

v1v/secret-analyzer.sh

Created May 16, 2023 20:20
Show Gist options
  • Save v1v/5e21a98dcfe0e98ae8475a665b31d059 to your computer and use it in GitHub Desktop.
Save v1v/5e21a98dcfe0e98ae8475a665b31d059 to your computer and use it in GitHub Desktop.
Download ZIP
Search for vault and github secrets in the given list of repositories
Raw
secret-analyzer.sh
#!/usr/bin/env bash
# Uses gh and the given list of repositories
# then it clone each repository and search for all the Jenkinsfiles or GitHub actions
# which use Vault or GitHub secrets using some regex.
REPO_LIST=$1
# Set GH Token
GITHUB_TOKEN=$(gh auth token)
export GH_PAGER=""
mkdir -p .repos
cd .repos
# Create header
echo "repo,jenkins-vault,action-with-vault,github-secrets,action-with-gh-secrets,jenkins-vault-secrets,vault-in-actions,gh-secrets,gh-secrets-in-actions" > report.csv
# Given the list of repositories
while read repo; do
echo "$repo"
## Download if it does not exist
if [ ! -d "$(basename "$repo")" ] ; then
if ! gh repo clone "$repo" "$(basename "$repo")" -- --branch main --single-branch --quiet --depth 1 ; then
gh repo clone "$repo" "$(basename "$repo")" -- --branch master --single-branch --quiet --depth 1
fi
fi
pushd "$(basename "$repo")" > /dev/null || exit
if ! git checkout main --quiet ; then
git checkout master --quiet
fi
## Search for CI pipelines 'secret/' or 'totp/'
pipeline_secrets_file=/tmp/pipeline.secrets
if [ -e "${pipeline_secrets_file}" ] ; then
rm "${pipeline_secrets_file}"
fi
for f in "Jenkinsfile" ".ci/Jenkinsfile" ; do
if [ -e "$f" ] ; then
echo " Search vault secrets in $f"
grep 'secret/' "$f" >> "${pipeline_secrets_file}"
grep 'totp/' "$f" >> "${pipeline_secrets_file}"
fi
done
if [ -d .ci ] ; then
for f in .ci/*.groovy ; do
if [ -e "$f" ] ; then
echo " Search vault secrets in $f"
grep 'secret/' "$f" >> "${pipeline_secrets_file}"
grep 'totp/' "$f" >> "${pipeline_secrets_file}"
fi
done
fi
pipeline_found=no
pipeline_secrets=
if [ -e "${pipeline_secrets_file}" ] ; then
pipeline_secrets=$(sed "s#\"#'#g" "${pipeline_secrets_file}" | sed "s#.*'\(.*\)'.*#\1#" | sort | uniq | paste -s -d';' -)
if [ $(cat "$pipeline_secrets_file" | wc -l) -gt 0 ] ; then
pipeline_found=yes
fi
fi
## Search for GitHub actions '{{ secrets.'
github_found=no
github_vault_found=no
github_action_secrets=
github_action_vault_secrets=
github_action_secrets_file=/tmp/github.secrets
github_action_vault_secrets_file=/tmp/github.vault.secrets
if [ -e "${github_action_secrets_file}" ] ; then
rm "${github_action_secrets_file}"
fi
if [ -e "${github_action_vault_secrets_file}" ] ; then
rm "${github_action_vault_secrets_file}"
fi
if [ -e .github/workflows ] ; then
for f in $(find .github/workflows -name '*.yml'); do
echo " Search vault and github secrets in $f"
grep 'secrets\.' "$f" >> "${github_action_secrets_file}"
grep 'secret/' "$f" >> "${github_action_vault_secrets_file}"
grep 'totp/' "$f" >> "${github_action_vault_secrets_file}"
done
if [ -e "${github_action_secrets_file}" ] ; then
github_action_secrets=$(sed 's#.*{{\(.*\)}}.*#\1#' "${github_action_secrets_file}" | sort | uniq | paste -s -d';' -)
if [ $(cat "$github_action_secrets_file" | wc -l) -gt 0 ] ; then
github_found=yes
fi
fi
if [ -e "${github_action_vault_secrets_file}" ] ; then
github_action_vault_secrets=$(sed "s#\"#'#g" "${github_action_vault_secrets_file}" | sed "s#.*\(secret/.*\) .*#\1#" | sed "s#.*\(totp/.*\) .*#\1#" | awk '{print $1}' | sort | uniq | paste -s -d';' -)
if [ $(cat "$github_action_vault_secrets_file" | wc -l) -gt 0 ] ; then
github_vault_found=yes
fi
fi
fi
## Search for GitHub secrets
echo " Search GitHub secrets in the repo"
github_secret_found=no
github_secrets=
if [ `gh secret list | wc -l` -gt 0 ] ; then
github_secrets=$(GH_PAGER="" gh secret list | awk '{print $1}' | paste -s -d';' -)
github_secret_found=yes
fi
popd > /dev/null || exit
echo "$repo,$pipeline_found,$github_vault_found,$github_found,$github_secret_found,$pipeline_secrets,$github_action_vault_secrets,$github_secrets,$github_action_secrets" | tee -a report.csv
done < "../$REPO_LIST"
@v1v
Copy link
Author

v1v commented May 17, 2023

And with Buildkite support:

#!/usr/bin/env bash
# Uses gh and the given list of repositories
# then it clone each repository and search for all the Jenkinsfiles or GitHub actions or Buildkite pipelines
# which use Vault or GitHub secrets using some regex.

REPO_LIST=$1

# Set GH Token
GITHUB_TOKEN=$(gh auth token)
export GH_PAGER=""

mkdir -p .repos
cd .repos

# Create header
echo "repo,jenkins-vault,action-with-vault,github-secrets,action-with-gh-secrets,buildkite-with-vault,jenkins-vault-secrets,vault-in-actions,gh-secrets,gh-secrets-in-actions,vault-in-buildkite" > report.csv
# Given the list of repositories
while read repo; do
  echo "$repo"

  ## Download if it does not exist
  if [ ! -d "$(basename "$repo")" ] ; then
    if ! gh repo clone "$repo" "$(basename "$repo")" -- --branch main --single-branch --quiet --depth 1 ; then
      gh repo clone "$repo" "$(basename "$repo")" -- --branch master --single-branch --quiet --depth 1
    fi
  fi

  pushd "$(basename "$repo")" > /dev/null || exit
  if ! git checkout main --quiet ; then
    git checkout master --quiet
  fi

  ## Search for CI pipelines 'secret/' or 'totp/'
  pipeline_secrets_file=/tmp/pipeline.secrets
  if [ -e "${pipeline_secrets_file}" ] ; then
    rm "${pipeline_secrets_file}"
  fi
  for f in "Jenkinsfile" ".ci/Jenkinsfile" ; do
    if [ -e "$f" ] ; then
      echo "   Search vault secrets in Jenkinsfile ($f)"
      grep 'secret/' "$f" >> "${pipeline_secrets_file}"
      grep 'totp/' "$f" >> "${pipeline_secrets_file}"
    fi
  done
  if [ -d .ci ] ; then
    for f in .ci/*.groovy ; do
      if [ -e "$f" ] ; then
        echo "   Search vault secrets in Jenkinsfile ($f)"
        grep 'secret/' "$f" >> "${pipeline_secrets_file}"
        grep 'totp/' "$f" >> "${pipeline_secrets_file}"
      fi
    done
  fi

  pipeline_found=no
  pipeline_secrets=
  if [ -e "${pipeline_secrets_file}" ] ; then
    pipeline_secrets=$(sed "s#\"#'#g" "${pipeline_secrets_file}" | sed "s#.*'\(.*\)'.*#\1#" | sort | uniq | paste -s -d';' -)
    if [ $(cat "$pipeline_secrets_file" | wc -l) -gt 0 ] ; then
      pipeline_found=yes
    fi
  fi

  ## Search for GitHub actions 'secrets.', 'secret/' or 'totp/'
  github_found=no
  github_vault_found=no
  github_action_secrets=
  github_action_vault_secrets=
  github_action_secrets_file=/tmp/github.secrets
  github_action_vault_secrets_file=/tmp/github.vault.secrets
  if [ -e "${github_action_secrets_file}" ] ; then
    rm "${github_action_secrets_file}"
  fi
  if [ -e "${github_action_vault_secrets_file}" ] ; then
    rm "${github_action_vault_secrets_file}"
  fi
  if [ -e .github/workflows ] ; then
    for f in $(find .github/workflows -name '*.yml'); do
      echo "   Search vault and github secrets in actions($f)"
      grep 'secrets\.' "$f" >> "${github_action_secrets_file}"
      grep 'secret/' "$f" >> "${github_action_vault_secrets_file}"
      grep 'totp/' "$f" >> "${github_action_vault_secrets_file}"
    done
    if [ -e "${github_action_secrets_file}"  ] ; then
      github_action_secrets=$(sed 's#.*{{\(.*\)}}.*#\1#' "${github_action_secrets_file}" | sort | uniq | paste -s -d';' -)
      if [ $(cat "$github_action_secrets_file" | wc -l) -gt 0 ] ; then
        github_found=yes
      fi
    fi
    if [ -e "${github_action_vault_secrets_file}"  ] ; then
      github_action_vault_secrets=$(sed "s#\"#'#g" "${github_action_vault_secrets_file}" | sed "s#.*\(secret/.*\) .*#\1#" | sed "s#.*\(totp/.*\) .*#\1#" |  awk '{print $1}' | sort | uniq | paste -s -d';' -)
      if [ $(cat "$github_action_vault_secrets_file" | wc -l) -gt 0 ] ; then
        github_vault_found=yes
      fi
    fi
  fi

  ## Search for GitHub secrets
  echo "   Search GitHub secrets in the repo"
  github_secret_found=no
  github_secrets=
  if [ `gh secret list | wc -l` -gt 0 ] ; then
    github_secrets=$(GH_PAGER="" gh secret list | awk '{print $1}' | paste -s -d';' -)
    github_secret_found=yes
  fi


  ## Search for Buildkite pipelines 'secret/' or 'totp/' or 'kv/'
  buildkite_secrets_file=/tmp/buildkite.secrets
  if [ -e "${buildkite_secrets_file}" ] ; then
    rm "${buildkite_secrets_file}"
  fi
  if [ -d .buildkite ] ; then
    for f in $(find .buildkite -type f); do
      if [ -e "$f" ] ; then
        echo "   Search vault secrets in buildkite($f)"
        grep 'secret/' "$f" >> "${buildkite_secrets_file}"
        grep 'totp/' "$f" >> "${buildkite_secrets_file}"
        grep 'kv/' "$f" >> "${buildkite_secrets_file}"
      fi
    done
  fi

  buildkite_found=no
  buildkite_secrets=
  if [ -e "${buildkite_secrets_file}" ] ; then
    buildkite_secrets=$(sed "s#\"#'#g" "${buildkite_secrets_file}" | sort | uniq | paste -s -d';' -)
    if [ $(cat "$buildkite_secrets_file" | wc -l) -gt 0 ] ; then
      buildkite_found=yes
    fi
  fi

  popd > /dev/null || exit

  echo "$repo,$pipeline_found,$github_vault_found,$github_found,$github_secret_found,$buildkite_found,$pipeline_secrets,$github_action_vault_secrets,$github_secrets,$github_action_secrets,$buildkite_secrets" | tee -a report.csv
done < "../$REPO_LIST"

@v1v
Copy link
Author

v1v commented May 22, 2023

Remove some grep/sed to avoid empty secrets in the GitHub actions and detect when composite actions are used so their secrets are added:

#!/usr/bin/env bash
# Uses gh and the given list of repositories
# then it clone each repository and search for all the Jenkinsfiles or GitHub actions
# which use Vault or GitHub secrets using some regex.

REPO_LIST=$1

# Set GH Token
GITHUB_TOKEN=$(gh auth token)
export GH_PAGER=""

mkdir -p .repos
cd .repos

# Create header
echo "repo,jenkins-vault,action-with-vault,github-secrets,action-with-gh-secrets,buildkite-with-vault,jenkins-vault-secrets,vault-in-actions,gh-secrets,gh-secrets-in-actions,vault-in-buildkite" > report.csv
# Given the list of repositories
while read repo; do
  echo "$repo"

  ## Download if it does not exist
  if [ ! -d "$(basename "$repo")" ] ; then
    if ! gh repo clone "$repo" "$(basename "$repo")" -- --branch main --single-branch --quiet --depth 1 ; then
      gh repo clone "$repo" "$(basename "$repo")" -- --branch master --single-branch --quiet --depth 1
    fi
  fi

  pushd "$(basename "$repo")" > /dev/null || exit

  git fetch
  git pull

  if ! git checkout main --quiet ; then
    git checkout master --quiet
  fi

  ## Search for CI pipelines 'secret/' or 'totp/'
  pipeline_secrets_file=/tmp/pipeline.secrets
  if [ -e "${pipeline_secrets_file}" ] ; then
    rm "${pipeline_secrets_file}"
  fi
  for f in "Jenkinsfile" ".ci/Jenkinsfile" ; do
    if [ -e "$f" ] ; then
      echo "   Search vault secrets in Jenkinsfile ($f)"
      grep 'secret/' "$f" >> "${pipeline_secrets_file}"
      grep 'totp/' "$f" >> "${pipeline_secrets_file}"
    fi
  done
  if [ -d .ci ] ; then
    for f in .ci/*.groovy ; do
      if [ -e "$f" ] ; then
        echo "   Search vault secrets in Jenkinsfile ($f)"
        grep 'secret/' "$f" >> "${pipeline_secrets_file}"
        grep 'totp/' "$f" >> "${pipeline_secrets_file}"
      fi
    done
  fi

  pipeline_found=no
  pipeline_secrets=
  if [ -e "${pipeline_secrets_file}" ] ; then
    pipeline_secrets=$(sed "s#\"#'#g" "${pipeline_secrets_file}" | sed "s#.*'\(.*\)'.*#\1#" | sort | uniq | paste -s -d';' -)
    if [ $(cat "$pipeline_secrets_file" | wc -l) -gt 0 ] ; then
      pipeline_found=yes
    fi
  fi

  ## Search for GitHub actions 'secrets.', 'secret/' or 'totp/'
  github_found=no
  github_vault_found=no
  github_action_secrets=
  github_action_vault_secrets=
  github_action_secrets_file=/tmp/github.secrets
  github_action_vault_secrets_file=/tmp/github.vault.secrets
  if [ -e "${github_action_secrets_file}" ] ; then
    rm "${github_action_secrets_file}"
  fi
  if [ -e "${github_action_vault_secrets_file}" ] ; then
    rm "${github_action_vault_secrets_file}"
  fi
  if [ -e .github/workflows ] ; then
    for f in $(find .github/workflows -name '*.yml'); do
      echo "   Search vault and github secrets in actions($f)"
      grep 'secrets\.' "$f" >> "${github_action_secrets_file}"
      grep 'secret/' "$f" >> "${github_action_vault_secrets_file}"
      grep 'totp/' "$f" >> "${github_action_vault_secrets_file}"

      # search for default secrets defined in the GitHub composite actions
      if grep -q 'elastic/apm-pipeline-library/.github/actions/opentelemetry' "$f" ; then
        echo 'secret/observability-team/ci/observability-ci/apm-credentials' >> "${github_action_vault_secrets_file}"
      fi
      if grep -q 'elastic/apm-pipeline-library/.github/actions/buildkite' "$f" ; then
        echo 'secret/observability-team/ci/buildkite-automation' >> "${github_action_vault_secrets_file}"
      fi
      if grep -q 'elastic/apm-pipeline-library/.github/actions/setup-npmrc' "$f" ; then
        echo 'secret/apm-team/ci/elastic-observability-npmjs' >> "${github_action_vault_secrets_file}"
      fi
    done
    if [ -e "${github_action_secrets_file}"  ] ; then
      github_action_secrets=$(sed 's#.*{{\(.*\)}}.*#\1#' "${github_action_secrets_file}" | sort | uniq | paste -s -d';' -)
      if [ $(cat "$github_action_secrets_file" | wc -l) -gt 0 ] ; then
        github_found=yes
      fi
    fi
    if [ -e "${github_action_vault_secrets_file}"  ] ; then
      #github_action_vault_secrets=$(sed "s#\"#'#g" "${github_action_vault_secrets_file}" | sed "s#.*\(secret/.*\) .*#\1#" | sed "s#.*\(totp/.*\) .*#\1#" |  awk '{print $1}' | sort | uniq | paste -s -d';' -)
      github_action_vault_secrets=$(sed "s#\"#'#g" "${github_action_vault_secrets_file}" | sort | uniq | paste -s -d';' -)
      if [ $(cat "$github_action_vault_secrets_file" | wc -l) -gt 0 ] ; then
        github_vault_found=yes
      fi
    fi
  fi

  ## Search for GitHub secrets
  echo "   Search GitHub secrets in the repo"
  github_secret_found=no
  github_secrets=
  if [ `gh secret list | wc -l` -gt 0 ] ; then
    github_secrets=$(GH_PAGER="" gh secret list | awk '{print $1}' | paste -s -d';' -)
    github_secret_found=yes
  fi

  ## Search for Buildkite pipelines 'secret/' or 'totp/' or 'kv/'
  buildkite_secrets_file=/tmp/buildkite.secrets
  if [ -e "${buildkite_secrets_file}" ] ; then
    rm "${buildkite_secrets_file}"
  fi
  if [ -d .buildkite ] ; then
    for f in $(find .buildkite -type f); do
      if [ -e "$f" ] ; then
        echo "   Search vault secrets in buildkite($f)"
        grep 'secret/' "$f" >> "${buildkite_secrets_file}"
        grep 'totp/' "$f" >> "${buildkite_secrets_file}"
        grep 'kv/' "$f" >> "${buildkite_secrets_file}"
      fi
    done
  fi

  buildkite_found=no
  buildkite_secrets=
  if [ -e "${buildkite_secrets_file}" ] ; then
    buildkite_secrets=$(sed "s#\"#'#g" "${buildkite_secrets_file}" | sort | uniq | paste -s -d';' -)
    if [ $(cat "$buildkite_secrets_file" | wc -l) -gt 0 ] ; then
      buildkite_found=yes
    fi
  fi

  popd > /dev/null || exit

  echo "$repo,$pipeline_found,$github_vault_found,$github_found,$github_secret_found,$buildkite_found,$pipeline_secrets,$github_action_vault_secrets,$github_secrets,$github_action_secrets,$buildkite_secrets" | tee -a report.csv
done < "../$REPO_LIST"

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment