cloud-bucket-recommendations.sh

cloud-bucket-recommendations.sh

#!/bin/bash

# Usage: bash <(curl -sL https://gist.githubusercontent.com/kienstra/fcc4c7d06a6e4d140b7fccf8d1106ced/raw/cloud-bucket-recommendations.sh) <project id>

if [ $# -ne 1 ]; then
	echo "Usage: $0 <project id>"
	exit 1
fi
PROJECT_ID=$1

TOKEN=$(gcloud auth print-access-token)
API="https://monitoring.googleapis.com/v3/projects/$PROJECT_ID/timeSeries"
DAYS_AGO_30=30
DAYS_AGO_90=90
METRIC="storage.googleapis.com/api/request_count"
OUTFILE=$(mktemp)

get_date() {
	if date -v -1d +%Y-%m-%dT%H:%M:%SZ >/dev/null 2>&1; then
		n_days=$(echo "$1" | grep -o '[0-9]\+')
		date -u -v "-${n_days}d" +"%Y-%m-%dT%H:%M:%SZ"
	else
		date -u -d "$1" +"%Y-%m-%dT%H:%M:%SZ"
	fi
}

NOW=$(get_date "0 days ago")
START_30=$(get_date "$DAYS_AGO_30 days ago")
START_90=$(get_date "$DAYS_AGO_90 days ago")

get_ops() {
	filter=$1
	start=$2
	days_ago=$3

	curl -s -H "Authorization: Bearer $TOKEN" "$API" \
		--get --data-urlencode "filter=$filter" \
		--data-urlencode "interval.startTime=$start" \
		--data-urlencode "interval.endTime=$NOW" \
		--data-urlencode "aggregation.perSeriesAligner=ALIGN_SUM" \
		--data-urlencode "aggregation.alignmentPeriod=$((days_ago*86400))s" \
		--data-urlencode "fields=timeSeries/points" \
		| jq -r '[.timeSeries[]?.points[].value.int64Value? | tonumber] | add // 0'
}

are_objects_standard() {
	obj_info=$1

	i=0
	for obj in $obj_info; do
		[ "$i" -eq 20 ] && break
		storage_class=$(gsutil stat "$obj" | awk 'tolower($0) ~ /storage class:/ {print toupper($NF); exit}')
		case "$storage_class" in
			STANDARD|REGIONAL|MULTI_REGIONAL) ;;
			*) return 1;
		esac
		i=$((i+1))
	done

	return 0
}

get_recommendation() {
	size_gb_num=$1
	obj_count=$2
	read_ops_30=$3
	read_ops_90=$4
	write_ops_30=$5
	write_ops_90=$6

	if (( size_gb_num < 10 )); then
		echo "None, too small to matter"
	elif (( obj_count < 1 )); then
		echo "None, no object in bucket"
	elif ! are_objects_standard "$OBJ_INFO"; then
		echo "None, contains non-standard storage class objects"
	elif [[ $read_ops_90 -eq 0 && $write_ops_90 -eq 0 ]]; then
		echo "Move to COLDLINE"
	elif [[ $read_ops_30 -eq 0 && $write_ops_30 -eq 0 ]]; then
		echo "Move to NEARLINE"
	else
		echo "None"
	fi
}

printf "READS_LAST_%dD\tWRITES_LAST_%dD\tREADS_LAST_%dD\tWRITES_LAST_%dD\tRECOMMENDATION\tBUCKET\tSIZE_GB\tobj_COUNT\tREGION\n" \
	"$DAYS_AGO_30" "$DAYS_AGO_30" "$DAYS_AGO_90" "$DAYS_AGO_90" > "$OUTFILE"

echo "Scanning buckets in project: $PROJECT_ID"

for BUCKET_URL in $(gsutil ls -p "$PROJECT_ID"); do
	BUCKET=$(echo "$BUCKET_URL" | sed 's|gs://||;s|/||g')
	echo "Checking bucket: $BUCKET"

	OBJ_LIST_RAW=$(gsutil ls -p "$PROJECT_ID" -l -r "gs://$BUCKET/**" 2>/dev/null \
		| awk 'NF==3 && $3 ~ /^gs:\/\// && $1 ~ /^[0-9]+$/')

	OBJ_INFO=$(echo "$OBJ_LIST_RAW" | awk '{print $3}')
	OBJ_COUNT=$(( $(echo "$OBJ_INFO" | wc -l | xargs) ))
	TOTAL_SIZE=$(echo "$OBJ_LIST_RAW" | awk '{s+=$1} END {print s}')
	[[ -z "$TOTAL_SIZE" ]] && TOTAL_SIZE=0

	METHODS='.*Read.*'
	FILTER="metric.type=\"$METRIC\" AND resource.labels.bucket_name=\"$BUCKET\" AND metric.labels.method=monitoring.regex.full_match(\"$METHODS\")"
	READ_OPS_30=$(get_ops "$FILTER" "$START_30" DAYS_AGO_30)
	READ_OPS_90=$(get_ops "$FILTER" "$START_90" DAYS_AGO_90)

	WRITE_METHODS='.*Insert.*|.*Update.*|.*Write.*|.*Compose.*|.*Upload.*|storage.objects.insert|storage.objects.update|storage.objects.compose|storage.objects.rewrite'
	FILTER_WRITE="metric.type=\"$METRIC\" AND resource.labels.bucket_name=\"$BUCKET\" AND metric.labels.method=monitoring.regex.full_match(\"$WRITE_METHODS\")"
	WRITE_OPS_30="$(get_ops "$FILTER_WRITE" "$START_30" DAYS_AGO_30)"
	WRITE_OPS_90="$(get_ops "$FILTER_WRITE" "$START_90" DAYS_AGO_90)"

	SIZE_GB=$(echo "scale=2; $TOTAL_SIZE/1024/1024/1024" | bc)
	SIZE_GB_NUM=$(( TOTAL_SIZE / 1024 / 1024 / 1024 ))
	RECOMMENDATION=$(get_recommendation "$SIZE_GB_NUM" $OBJ_COUNT "$READ_OPS_30" "$READ_OPS_90" "$WRITE_OPS_30" "$WRITE_OPS_90")
	REGION=$(gcloud storage buckets describe "gs://$BUCKET" \
		--project="$PROJECT_ID" \
		--format='value(location)')
	REGION=${REGION:-global}

	printf "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n" \
		"$READ_OPS_30" "$WRITE_OPS_30" "$READ_OPS_90" "$WRITE_OPS_90" "$RECOMMENDATION" "$BUCKET" "$SIZE_GB" "$OBJ_COUNT" "$REGION" \
		>> "$OUTFILE"
done

echo
echo "Recommendation report for project ${PROJECT_ID}:"
column -t -s $'\t' "$OUTFILE"