Skip to content

Instantly share code, notes, and snippets.

@ilatypov

ilatypov/bd.sh

Created September 12, 2019 17:12
Show Gist options
  • Save ilatypov/96018a7d3256357e43e0af4c9c0832c0 to your computer and use it in GitHub Desktop.
Save ilatypov/96018a7d3256357e43e0af4c9c0832c0 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
bd.sh
#! /bin/bash
function usage() {
echo "Usage: $0 [-r URL] [-k] (--projects|--scans|--purge-old-overlapping-maven-scans) [TOKEN|USER:PASS]
For example,
\$ export BDSERVER="https://COMPANY.blackducksoftware.com"
\$ read -rsp \"Password: \" BDPASS
\$ export BDCRED="USER:\${BDPASS}"
\$ $0 --projects
" >&2
if (( $# )) ; then
echo >&2
echo "$*" >&2
fi
exit 1
}
if ! "${SHELL}" -c 'mapfile' < /dev/null > /dev/null 2>&1 ; then
function mapfile() {
local trim=0
if [[ "$1" == "-t" ]] ; then
trim=1
shift
fi
local varname="$1"; shift
source /dev/stdin <<<"${varname}=()"
local line
while read -r line ; do
if (( trim )) ; then
while [[ "${line}" =~ ^(.*)$'\n'$ ]] ; do
line="${BASH_REMATCH[1]}"
done
fi
source /dev/stdin <<<"${varname}+=(\"\${line}\")"
done
}
export -f mapfile
fi
set -e
bdserver="${BDSERVER}"
tmproot="${TMPDIR:-/tmp}"
tmproot="${tmproot%/}"
tmp="${tmproot}/bd"
rm -rf "${tmp}"
mkdir -p "${tmp}"
mitmcert="${tmp}/zscaler-ca.pem"
cat > "${mitmcert}" <<"EOT"
##
## The Zscaler's Certification Authority (self-signed) certificate.
##
## The following command from a workstation's registry generated the
## Zscaler part of the bundle. The workstation appears to have received
## the CA via a group policy update.
##
## powershell "Get-ChildItem -Path Cert:\LocalMachine\Root | where {$_.Issuer -like \"*cn=zscaler*\" } | foreach {$content = \"-----BEGIN CERTIFICATE-----`r`n$([System.Convert]::ToBase64String($_.RawData, 'InsertLineBreaks'))`r`n-----END CERTIFICATE-----\"; $content | Out-File -FilePath \"zscaler-ca.pem\" -Encoding ascii}"
##
##
E=support@zscaler.com, CN=Zscaler Root CA, OU=Zscaler Inc., O=Zscaler Inc., L=San Jose, S=California, C=US
==========================================================================================================
-----BEGIN CERTIFICATE-----
MIIE0zCCA7ugAwIBAgIJANu+mC2Jt3uTMA0GCSqGSIb3DQEBCwUAMIGhMQswCQYDVQQGEwJVUzET
MBEGA1UECBMKQ2FsaWZvcm5pYTERMA8GA1UEBxMIU2FuIEpvc2UxFTATBgNVBAoTDFpzY2FsZXIg
SW5jLjEVMBMGA1UECxMMWnNjYWxlciBJbmMuMRgwFgYDVQQDEw9ac2NhbGVyIFJvb3QgQ0ExIjAg
BgkqhkiG9w0BCQEWE3N1cHBvcnRAenNjYWxlci5jb20wHhcNMTQxMjE5MDAyNzU1WhcNNDIwNTA2
MDAyNzU1WjCBoTELMAkGA1UEBhMCVVMxEzARBgNVBAgTCkNhbGlmb3JuaWExETAPBgNVBAcTCFNh
biBKb3NlMRUwEwYDVQQKEwxac2NhbGVyIEluYy4xFTATBgNVBAsTDFpzY2FsZXIgSW5jLjEYMBYG
A1UEAxMPWnNjYWxlciBSb290IENBMSIwIAYJKoZIhvcNAQkBFhNzdXBwb3J0QHpzY2FsZXIuY29t
MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqT7STSxZRTgEFFf6doHajSc1vk5jmzmM
6BWuOo044EsaTc9eVEV/HjH/1DWzZtcrfTj+ni205apMTlKBW3UYR+lyLHQ9FoZiDXYXK8poKSV5
+Tm0Vls/5Kb8mkhVVqv7LgYEmvEY7HPY+i1nEGZCa46ZXCOohJ0mBEtB9JVlpDIO+nN0hUMAYYdZ
1KZWCMNf5J/aTZiShsorN2A38iSOhdd+mcRM4iNL3gsLu99XhKnRqKoHeH83lVdfu1XBeoQzz5V6
gA3kbRvhDwoIlTBeMa5l4yRdJAfdpkbFzqiwSgNdhbxTHnYYorDzKfr2rEFMdsMU0DHdeAZf711+
1CunuQIDAQABo4IBCjCCAQYwHQYDVR0OBBYEFLm33UrNww4Mhp1d3+wcBGnFTpjfMIHWBgNVHSME
gc4wgcuAFLm33UrNww4Mhp1d3+wcBGnFTpjfoYGnpIGkMIGhMQswCQYDVQQGEwJVUzETMBEGA1UE
CBMKQ2FsaWZvcm5pYTERMA8GA1UEBxMIU2FuIEpvc2UxFTATBgNVBAoTDFpzY2FsZXIgSW5jLjEV
MBMGA1UECxMMWnNjYWxlciBJbmMuMRgwFgYDVQQDEw9ac2NhbGVyIFJvb3QgQ0ExIjAgBgkqhkiG
9w0BCQEWE3N1cHBvcnRAenNjYWxlci5jb22CCQDbvpgtibd7kzAMBgNVHRMEBTADAQH/MA0GCSqG
SIb3DQEBCwUAA4IBAQAw0NdJh8w3NsJu4KHuVZUrmZgIohnTm0j+RTmYQ9IKA/pvxAcA6K1i/LO+
Bt+tCX+C0yxqB8qzuo+4vAzoY5JEBhyhBhf1uK+P/WVWFZN/+hTgpSbZgzUEnWQG2gOVd24msex+
0Sr7hyr9vn6OueH+jj+vCMiAm5+ukd7lLvJsBu3AO3jGWVLyPkS3i6Gf+rwAp1OsRrv3WnbkYcFf
9xjuaf4z0hRCrLN2xFNjavxrHmsH8jPHVvgc1VD0Opja0l/BRVauTrUaoW6tE+wFG5rEcPGS80jj
HK4SpB5iDj2mUZH1T8lzYtuZy0ZPirxmtsk3135+CKNa2OCAhhFjE0xd
-----END CERTIFICATE-----
EOT
rm -rf "${tmp}/certs"
mkdir -p "${tmp}/certs"
certname=$(openssl x509 -hash -noout -in "${mitmcert}")
ln -s "${mitmcert}" "${tmp}/certs/${certname}.0"
copts=(--capath "/etc/ssl/certs:${tmp}/certs")
while (( $# )) ; do
case "$1" in
-r)
shift
bdserver="$1"; shift || usage
;;
-k)
copts+=($1)
shift
;;
*)
break
;;
esac
done
(( ${#bdserver} )) || usage "Missing the server URL"
command="$1" ; shift || usage
cred="$1"; shift || cred="${BDCRED}"
(( ! $# )) || usage
if [[ "${cred}" =~ : ]] ; then
user="${cred%%:*}"
pass="${cred#*:}"
if (( ! ${#pass} )) ; then
if (( ${#BDPASS} )) ; then
pass="${BDPASS}"
else
read -rsp "Password: " pass
echo
fi
fi
echo "Logging in as ${user}..."
http_code=$(curl -isS "${copts[@]}" --write-out "%{http_code}" -X POST -o "${tmp}/login-pass.txt" \
--data-urlencode "j_username=${user}" \
--data-urlencode "j_password=${pass}" \
"${bdserver}/j_spring_security_check")
if [[ "${http_code}" != "200" && "${http_code}" != "204" ]] ; then
echo "HTTP response code ${http_code}"
cat "${tmp}/login-pass.txt"
echo
exit 1
fi
bearertoken=$(sed -ne 's/^Set-Cookie: AUTHORIZATION_BEARER=\([^;[:space:]]*\).*/\1/p' "${tmp}/login-pass.txt")
csrftoken=$(sed -ne 's/^X-CSRF-TOKEN: \([^;[:space:]]*\).*/\1/p' "${tmp}/login-pass.txt")
else
token="${cred}"
echo "Logging in with the token..."
# https://github.com/blackducksoftware/blackduck-common/blob/master/src/main/java/com/synopsys/integration/blackduck/rest/ApiTokenBlackDuckHttpClient.java
http_code=$(curl -isS "${copts[@]}" --write-out "%{http_code}" -X POST \
-H "Authorization: token ${token}" -o "${tmp}/login-token.txt" \
"${bdserver}/api/tokens/authenticate")
if [[ "${http_code}" != "200" && "${http_code}" != "204" ]] ; then
echo "HTTP response code ${http_code}"
cat "${tmp}/login-token.txt"
echo
exit 1
fi
body=$(sed -e '1,/^[[:space:]]*$/d' "${tmp}/login-token.txt")
bearertoken=$(jq -r ".bearerToken" <<< "${body}")
csrftoken=$(sed -ne 's/^X-CSRF-TOKEN: \([^;[:space:]]*\).*/\1/p' "${tmp}/login-token.txt")
fi
copts+=(\
-H "Cookie: AUTHORIZATION_BEARER=${bearertoken}" \
# -H "Authorization: Bearer ${bearertoken}" \
-H "X-CSRF-TOKEN: ${csrftoken}" \
)
case "${command}" in
--projects)
projstart=0
projlimit=100
while : ; do
echo "Getting a list of ${projlimit} projects from ${projstart}..."
http_code=$(curl -sS "${copts[@]}" --write-out "%{http_code}" -o "${tmp}/projects-${projstart}.json" \
"${bdserver}/api/projects" \
-G --data-urlencode "offset=${projstart}" --data-urlencode "limit=${projlimit}")
if [[ "${http_code}" != "200" ]] ; then
echo "HTTP response code ${http_code}"
echo
jq . "${tmp}/projects-${projstart}.json" || cat "${tmp}/projects-${projstart}.json"
echo
exit 1
fi
# jq . "${tmp}/projects-${projstart}.json"
numitems=$(jq ".items | length" "${tmp}/projects-${projstart}.json")
echo "Got ${numitems} projects starting from ${projstart}."
(( numitems )) || break
mapfile -t projs < <(jq -r '.items[] | ._meta.href + " " + .name' "${tmp}/projects-${projstart}.json")
for projrefname in "${projs[@]}" ; do
projref="${projrefname%% *}"
projname="${projrefname#* }"
echo " Project <<${projname}>>"
mapfile -t versions < <(curl -sS "${copts[@]}" "${projref}/versions" | jq -r '.items[] | ._meta.href + " " + .versionName')
for versionrefname in "${versions[@]}" ; do
verref="${versionrefname%% *}"
vername="${versionrefname#* }"
echo " Version <<${vername}>>"
mapfile -t trees < <(curl -sS "${copts[@]}" "${verref}/source-trees" | jq -r '.items[] | .name')
for tree in "${trees[@]}" ; do
echo " Source tree <<${tree}>>"
done
done
done
((projstart+=projlimit))
done
;;
--scans|--purge-old-overlapping-maven-scans)
scanstart=0
scanlimit=100
while : ; do
echo "Getting a list of ${scanlimit} scans from ${scanstart}..."
http_code=$(curl -sS "${copts[@]}" --write-out "%{http_code}" -o "${tmp}/scans-${scanstart}.json" \
"${bdserver}/api/codelocations" \
-G --data-urlencode "offset=${scanstart}" --data-urlencode "limit=${scanlimit}" \
--data-urlencode "sort=updatedAt" --data-urlencode "order=asc" --data-urlencode "includeErrors=true")
if [[ "${http_code}" != "200" ]] ; then
echo "HTTP response code ${http_code}"
echo
jq . "${tmp}/scans-${scanstart}.json" || cat "${tmp}/scans-${scanstart}.json"
echo
exit 1
fi
# jq . "${tmp}/scans-${scanstart}.json"
numitems=$(jq ".items | length" "${tmp}/scans-${scanstart}.json")
echo "Got ${numitems} scans starting from ${scanstart}."
(( numitems )) || break
mapfile -t scans < <(jq -r \
'.items[] | ._meta.href + " " + (.scanSize|tostring) + " " + .name + ": " + .updatedAt + " " + ([.status[].status] | join(" "))' \
"${tmp}/scans-${scanstart}.json")
for scanrefname in "${scans[@]}" ; do
scanref="${scanrefname%% *}"
scansizename="${scanrefname#* }"
scansize="${scansizename%% *}"
scannamest="${scansizename#* }"
scanname="${scannamest%%: *}"
scanst="${scannamest#*: }"
laststatus="${scanst##* }"
updatets="${scanst%% *}"
scanst="${scanst#* }"
case "${command}" in
--scans)
echo " Scan <<${scanname}>>, size ${scansize}, date ${updatets}, statuses ${scanst}"
;;
--purge-old-overlapping-maven-scans)
# https://github.com/blackducksoftware/synopsys-detect/issues/66
if [[ "${scanname}" =~ [[:space:]]BOM$ ]] && (( !scansize )) ; then
echo " Suspicious scan ${scanname}, size ${scansize}, date ${updatets}, statuses ${scanst}"
case "${laststatus}" in
"COMPLETED"|"UNSTARTED")
delref="${scanref//\/api\///api/v1/composite/}"
echo " DELETE ${delref}"
http_code=$(curl -isS "${copts[@]}" --write-out "%{http_code}" -o "${tmp}/delete-scan-${updatets}-${scanref##*/}.txt" \
"${delref}" \
-X DELETE)
if [[ "${http_code}" != "204" ]] ; then
echo "HTTP response code ${http_code}"
echo
cat "${tmp}/delete-scan-${updatets}-${scanref##*/}.txt"
echo
# exit 1
fi
;;
esac
fi
;;
esac
done
((scanstart+=scanlimit))
done
;;
--notifications)
# This returns 403
notifstart=0
notiflimit=100
echo "Getting ${notiflimit} notifications starting from ${notifstart}..."
echo "Bearer token ${bearertoken}"
echo "CSRF token ${csrftoken}"
http_code=$(curl -sS "${copts[@]}" --write-out "%{http_code}" -o "${tmp}/notifications-${notifstart}.json" \
"${bdserver}/api/notifications" \
-G --data-urlencode "offset=${notifstart}" --data-urlencode "limit=${notiflimit}")
if [[ "${http_code}" != "200" ]] ; then
echo "HTTP response code ${http_code}"
echo
jq . "${tmp}/notifications-${notifstart}.json" || cat "${tmp}/notifications-${notifstart}.json"
echo
exit 1
fi
jq . "${tmp}/notifications-${notifstart}.json"
;;
*)
usage
;;
esac
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment