aws-cli-tldr

filter-log-events.sh

#!/usr/bin/env bash

LOG_GROUP_NAME=${1-} # API-ResourceFunction
MY_QUERY="${@:2}"
START_TIME=$(date --date="${START_TIME:-30 minutes ago}" "+%s000")
END_TIME=$(date "+%s000")

if [ -z "${LOG_GROUP_NAME}" ]; then
  aws logs describe-log-groups --query 'logGroups[].logGroupName'
  exit 0;
else
  LOG_GROUP=$(aws logs describe-log-groups --query 'logGroups[].logGroupName' | \
    jq -r '. | map(select(test("'$LOG_GROUP_NAME'"))) | .[0]')

  if [ -z "$LOG_GROUP" ]; then
    echo "$LOG_GROUP_NAME not found"
    exit 1
  fi
fi


RESULTS=$(aws logs filter-log-events \
 --log-group-name $LOG_GROUP \
 --start-time $START_TIME \
 --end-time $END_TIME \
 --filter-pattern "$MY_QUERY" | jq -r '.events')

if [ "$RESULTS" != "[]" ]; then
  case $VARIANT in
    0)
      START_TIME=$(echo "$RESULTS" | jq -r '.[0].timestamp')
      END_TIME=$(echo "$RESULTS" | jq -r '.[-1].timestamp')
      aws logs filter-log-events \
        --log-group-name $LOG_GROUP \
        --start-time $START_TIME \
        --end-time $END_TIME | jq -r .events[].message;;
    *)
      # # Variant ii
      LOG_STREAM=$(echo "$RESULTS" | jq -r '.[-1].logStreamName')
      aws logs filter-log-events \
        --log-group-name $LOG_GROUP \
        --log-stream-names $LOG_STREAM \
        | jq -r .events[].message;;
    esac
else
  echo 'No results!!' && false
fi

find_PulpDropbox_logs.sh

#!/usr/bin/env bash
MY_QUERY='"app-5.2.2-1.fc32.x86_64.rpm"'
GROUP="/aws/lambda/PulpDropBox-DropBoxLambda-e6JscLnDec8b"
aws logs filter-log-events \
  --log-group-name $GROUP \
  --start-time $(date --date="1 day ago" +%s000) \
  --filter-pattern "$MY_QUERY"

STREAM='2021/09/28/[$LATEST]dc262970bde94a9d86a00ba91f5e62a4'
aws logs filter-log-events --log-stream-names $STREAM --log-group-name $GROUP --query "events[].message"

find_s3_object.sh

BUCKET=corporatefileimporter-corpfilebucket-18xak5bfah5x5
aws s3api list-objects --bucket $BUCKET | jq -c '.Contents[].Key' | grep "Site-Contract-Line-Item.txt" | sort | tail

find_tie_cloud_purchases.sh

LOG_GROUP_NAME="/aws/lambda/API-ResourceFunction-3YD4I82B6Q9C"

START_DATE_TIME=$(date --date="1 month ago" +%s000)
aws logs filter-log-events \
   --log-group-name $LOG_GROUP_NAME \
   --start-time $START_DATE_TIME \
   --filter-pattern "purchase_id" | jq -c '.events[] | select(.message | test("Sending request:")) | .message | ltrimstr("Sending request: ") | fromjson | .parameters.purchase_id' > /tmp/FILENAME


python <<EOF

with open('/tmp/FILENAME') as fid:
    lines = fid.readlines()

import re

reg = re.compile('(^([0-9A-Za-z][0-9]{5}[0-9A-F]{4})|([0-9A-Za-z][0-9]{5}[A-Z]{2}[0-9]{5})$)|(^__unknown__$)')

for line in lines:
    l = line.replace('"', '').strip()
    m = reg.match(l)
    print(f"{m} {l}")
EOF

get_site_pillar.sh

account="dev"
site="0290912"

BUCKET_NAME=$(aws s3api list-buckets | jq -r '.Buckets  | map(select(.Name | test("'"$account"'-salt"))) | .[0].Name')

OBJECT_KEY=$(aws s3api list-objects --bucket $BUCKET_NAME | jq -r '.Contents | map(select(.Key | test("'$site'"))) | .[0].Key')

FILENAME=/tmp/$(basename $OBJECT_KEY)

aws s3api head-object --bucket $BUCKET_NAME --key $OBJECT_KEY
aws s3api get-object --bucket $BUCKET_NAME --key $OBJECT_KEY $FILENAME

echo $FILENAME
cat $FILENAME

notes.sh

#!/usr/bin/env bash

# Verify an email address so it can recieve email from our sandbox SES

EMAILADDR=${1}

aws ses verify-email-identity --email-address ${EMAILADDR}
#!/usr/bin/env bash

tmpfile=$(mktemp /tmp/my-function-XXXX.zip)
rm $tmpfile

CODE_LOC=/home/jbrinkma/dev/
cd $CODE_LOC || exit 1

LAMBDA_NAME=$(aws lambda list-functions \
    | jq -r '.Functions[].FunctionName | select(test("SNC"))')

pip install --upgrade -r requirements.txt --target .
zip $tmpfile -r ./* &> /dev/null

RESULT=$(aws lambda update-function-code \
    --function-name $LAMBDA_NAME \
    --zip-file "fileb://$tmpfile" \
    --publish)

echo "$RESULT" | jq -r .LastUpdateStatus

# aws lambda update-function-configuration \
#     --function-name $LAMBDA_NAME \
#     --timeout 30 \
#     --memory-size 256 | jq -r .LastUpdateStatus

rm $tmpfile
#!/usr/bin/env bash

TOPIC_ARN=${1-}
MY_EMAIL=${2-}

if [ -z "${TOPIC_ARN}" ]; then
  aws sns list-topics --query "Topics[].TopicArn"
  exit 0;
else
  TOPIC_ARN=$(aws sns list-topics \
    --query "Topics[?contains(TopicArn, '"$TOPIC_ARN"')].TopicArn" \
    --output text)
fi

aws sns subscribe \
  --topic-arn $TOPIC_ARN \
  --protocol email \
  --notification-endpoint $MY_EMAIL
#!/usr/bin/env bash

if [ ! -z ${1} ]; then
  QUEUE_URL=$(aws sqs list-queues \
    | jq -r '.QueueUrls[] | select(test("'$1'"))');
else
  aws sqs list-queues --query 'QueueUrls';
  exit 1
fi

SITE_ID=${2:-0000057}

MSG=$(echo '{
    "type": "checkout",
    "source": {
      "room": "8204", 
      "occupied": false, 
      "reservations": []
    }, 
    "version": "1.0.0", 
    "integration_timestamp": "2021-11-04T16:08:44.092000+00:00", 
    "event_id": "JAKE IS TESTING THINGS", 
    "site_id": "0000057"
  }' | jq -c '.' )

aws sqs send-message --message-group-id $SITE_ID \
  --queue-url $QUEUE_URL \
  --message-body "$MSG"

EVENT=$(cat << '__END__' | jq -c
[
  {
    "EventBusName": "Late-Checkout",
    "Source": "/late-checkout",
    "DetailType": "cloud.late-checkout.failed.v1",
    "Detail": "{\"site_id\": \"0279927\", \"room_number\": \"0010\", \"id\": \"friday-testing\"}"
  }
]
__END__
)

aws events put-events --entries "$EVENT"
GROUP_NAME="/aws/lambda/API-ResourceFunction-GSEWHYPFOAUH"

aws logs filter-log-events --log-group-name $GROUP_NAME --start-time $(date --date="1 day ago" +%s000) --filter-pattern "system.resource" | jq -c '.events[].message | select(test("^Send Event: ")) | sub("^Send Event: .*Detail.*{"; "{") | sub("}.*"; "}")| fromjson'
LOG_GROUP_NAME="/aws/lambda/API-ResourceFunction-3YD4I82B6Q9C"

START_DATE_TIME=$(date --date="1 month ago" +%s000)
aws logs filter-log-events \
   --log-group-name $LOG_GROUP_NAME \
   --start-time $START_DATE_TIME \
   --filter-pattern "purchase_id" | jq -c '.events[] | select(.message | test("Sending request:")) | .message | ltrimstr("Sending request: ") | fromjson | .parameters.purchase_id' > /tmp/FILENAME


python <<EOF

with open('/tmp/FILENAME') as fid:
    lines = fid.readlines()

import re

reg = re.compile('(^([0-9A-Za-z][0-9]{5}[0-9A-F]{4})|([0-9A-Za-z][0-9]{5}[A-Z]{2}[0-9]{5})$)|(^__unknown__$)')

for line in lines:
    l = line.replace('"', '').strip()
    m = reg.match(l)
    print(f"{m} {l}")
EOF
#!/usr/bin/env bash
MY_QUERY='"app-5.2.2-1.fc32.x86_64.rpm"'
GROUP="/aws/lambda/PulpDropBox-DropBoxLambda-e6JscLnDec8b"
aws logs filter-log-events \
  --log-group-name $GROUP \
  --start-time $(date --date="1 day ago" +%s000) \
  --filter-pattern "$MY_QUERY"

STREAM='2021/09/28/[$LATEST]dc262970bde94a9d86a00ba91f5e62a4'
aws logs filter-log-events --log-stream-names $STREAM --log-group-name $GROUP --query "events[].message"

pull_event_bridge_events.sh

GROUP_NAME="/aws/lambda/API-ResourceFunction-GSEWHYPFOAUH"

aws logs filter-log-events --log-group-name $GROUP_NAME --start-time $(date --date="1 day ago" +%s000) --filter-pattern "system.resource" | jq -c '.events[].message | select(test("^Send Event: ")) | sub("^Send Event: .*Detail.*{"; "{") | sub("}.*"; "}")| fromjson'

put_auditlog_sqs_events.sh

#!/usr/bin/env bash
# SCENARIO: Test the integration from AuditLogging SQS to DBWriter Lambda
SOURCE_SQS_QUEUE=Pms-Audit-Logging-AuditLoggingQueue
DESTINATION_LAMBDA=Pms-Audit-Logging-DbWriterFunction
set -eoux pipefail

timestamp=$(date "+%s000")

EVENTS=$(cat << EOF | jq -c
{
  "correlation_id": "38062a15-2f34-4e12-af36-047acdc12249",
  "timestamp": 1640557342655,
  "site_id": "0283502",
  "room_id": "304",
  "event_type": "look",
  "event_source": "api.cloud",
  "event_body": null
}
EOF
)

QUEUE=$(aws sqs list-queues | jq -r '.QueueUrls[] | select(test("'$SOURCE_SQS_QUEUE'"))')
GROUP=$(aws logs describe-log-groups | jq -r '.logGroups[].logGroupName | select(test("'$DESTINATION_LAMBDA'"))')

aws sqs send-message \
  --queue-url $QUEUE \
  --message-body "$EVENTS"

sleep 60

aws logs tail --since '3m' --format short "$GROUP"

put_log_events.sh

#!/usr/bin/env bash
# SCENARIO: Test the integration from CloudWatch to our Log Processor Lambda
# GIVEN: The Cloudwatch logstream exists
#  AND: The FilterSubscription is in place to destination function
# WHEN: I submit an event which matchs the filter to the SOURCE log group
# THEN: I will see my DESTINATION lambda was triggered
SOURCE_LOG_GROUP=Site-WS-EventFunction
DESTINATION_LAMBDA=Pms-Audit-Logging-LogProcessorFunction

timestamp=$(date "+%s000")
message='Event: { \"type\": \"link-status\" }'

EVENTS=$(cat << EOF | jq -c
[
  {
    "timestamp": $timestamp,
    "message": "$message"
  }
]
EOF
)

GROUP=$(aws logs describe-log-groups | jq -r '.logGroups[].logGroupName | select(test("Site-WS-EventFunction"))')
STREAM=$(date "+%Y%m%d")
aws logs create-log-stream --log-group-name $GROUP --log-stream-name $STREAM

aws logs put-log-events \
  --log-group-name $GROUP \
  --log-stream-name $STREAM \
  --log-events "$EVENTS"

sleep 10
aws logs delete-log-stream --log-group-name $GROUP --log-stream-name $STREAM

GROUP=$(aws logs describe-log-groups | jq -r '.logGroups[].logGroupName | select(test("LogProcessorFunction"))')
aws logs tail --since '3m' --format short "$GROUP"

pyproject.toml

[tool.poetry]
name = "aws-cli-tldr"
version = "0.1.0"
description = ""
authors = ["Your Name <you@example.com>"]

[tool.poetry.dependencies]
python = "^3.9"
aws-sam-cli = "^1.35.0"

[tool.poetry.dev-dependencies]
black = "^19.10b0"
flake8 = "^4.0.1"
mypy = "^0.910"
isort = "^5.10.1"
pytest = "^6.2.5"
pytest-cov = "^3.0.0"
pytest-mock = "^3.6.1"

[build-system]
requires = ["poetry-core>=1.0.0"]
build-backend = "poetry.core.masonry.api"

query-log-insights.sh

#!/usr/bin/env bash

# https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/CWL_QuerySyntax.html
LOG_GROUP_NAME=${1-}
MY_QUERY="${@:2}"
START_TIME=$(date --date="${START_TIME:-30 minutes ago}" "+%s000")
END_TIME=$(date "+%s000")

if [ -z "${MY_QUERY}" ]; then
  echo 'example: '"'"'filter @message like "0284062" and @message like "PATCH"'"'"''
  echo 'example: '"'"'filter @requestId like /fcd09029-0e22-4f57-826e-a64ccb385330/'"'"''
  exit 1;
fi

if [ -z "${LOG_GROUP_NAME}" ]; then
  aws logs describe-log-groups --query 'logGroups[].logGroupName'
  exit 0;
else
  LOG_GROUP=$(aws logs describe-log-groups --query 'logGroups[].logGroupName' | \
    jq -r '. | map(select(test("'$LOG_GROUP_NAME'"))) | .[0]')

  if [ -z "$LOG_GROUP" ]; then
    echo "$LOG_GROUP_NAME not found"
    exit 1
  fi
fi

QUERY_ID=$(aws logs start-query \
 --log-group-names $LOG_GROUP\
 --start-time $START_TIME \
 --end-time $END_TIME \
 --query-string 'fields @timestamp, @message | '"$MY_QUERY"' | sort @timestamp desc | limit 20' | jq -r .queryId)


STATUS="Running"
while [ $STATUS = "Running" ]; do
  sleep 1;
  STATUS=$(aws logs get-query-results --query-id $QUERY_ID | jq -r '.status');
done


aws logs get-query-results --query-id $QUERY_ID \
  | jq -r '.results[] | map(select(.field == "@timestamp" or .field == "@message") | .value) | @tsv'

submit-groupservices-event.sh

STATEMACHINE=arn:aws:states:us-east-1:952956287553:stateMachine:Group-Services-TerminalLineupChanged
EVENT=$(cat <<__JSON__
{
  "version": "0",
  "id": "3cccd183-bd5e-d43e-95c5-3415b971795e",
  "detail-type": "cloud.terminal.updated.v1",
  "source": "/app-api",
  "account": "952956287553",
  "time": "2022-01-10T21:20:32Z",
  "region": "us-east-1",
  "resources": [],
  "detail": {
    "site_id": "0279927",
    "terminal_id": "a8:23:fe:58:58:e9",
    "correlation_id": "bd0142d6-cf4a-4d4e-866d-db789f014292"
  }
}
__JSON__
)


aws stepfunctions start-execution --state-machine-arn $STATEMACHINE --input "$EVENT"

submit_disney_events.sh

#!/usr/bin/env bash

if [ ! -z ${1} ]; then
  QUEUE_URL=$(aws sqs list-queues \
    | jq -r '.QueueUrls[] | select(test("'$1'"))');
else
  aws sqs list-queues --query 'QueueUrls';
  exit 1
fi

SITE_ID=${2:-0000057}

MSG=$(echo '{
    "type": "checkout",
    "source": {
      "room": "8204", 
      "occupied": false, 
      "reservations": []
    }, 
    "version": "1.0.0", 
    "integration_timestamp": "2021-11-04T16:08:44.092000+00:00", 
    "event_id": "JAKE IS TESTING THINGS", 
    "site_id": "0000057"
  }' | jq -c '.' )

aws sqs send-message --message-group-id $SITE_ID \
  --queue-url $QUEUE_URL \
  --message-body "$MSG"

submit_latecheckout_eventbus.sh

EVENT=$(cat << '__END__' | jq -c
[
  {
    "EventBusName": "Late-Checkout",
    "Source": "/late-checkout",
    "DetailType": "cloud.late-checkout.failed.v1",
    "Detail": "{\"site_id\": \"0279927\", \"room_number\": \"0010\", \"id\": \"friday-testing\"}"
  }
]
__END__
)

aws events put-events --entries "$EVENT"

subscribe_sns_queue.sh

#!/usr/bin/env bash

TOPIC_ARN=${1-}
MY_EMAIL=${2-}

if [ -z "${TOPIC_ARN}" ]; then
  aws sns list-topics --query "Topics[].TopicArn"
  exit 0;
else
  TOPIC_ARN=$(aws sns list-topics \
    --query "Topics[?contains(TopicArn, '"$TOPIC_ARN"')].TopicArn" \
    --output text)
fi

aws sns subscribe \
  --topic-arn $TOPIC_ARN \
  --protocol email \
  --notification-endpoint $MY_EMAIL

tail_lambda_func_logs.sh

#!/usr/bin/env bash

if [ -z "${1}" ]; then
  aws lambda list-functions | jq '.Functions[].FunctionName' | sort;
  exit 1;
fi

LAMBDA=$(aws lambda list-functions \
  | jq -r '.Functions[].FunctionName | select(test("'"${1}"'"))')

if [ -z "${2}" ]; then
  sam logs -n $LAMBDA --tail
else 
  sam logs -n $LAMBDA --tail "${2}"
fi

trigger_pillar_generation.sh

LOG_GROUP_NAME="/aws/lambda/System-Event-Processor-QueueProcessor-xbOFMm15h1Gq"

aws logs filter-log-events --log-group-name $LOG_GROUP_NAME --start-time $(date --date="1 day ago" +%s000) --filter-pattern "0289045" | jq -c '.events[]'
aws logs filter-log-events --log-group-name $LOG_GROUP_NAME --start-time $(date --date="1 day ago" +%s000) --filter-pattern "Error" | jq -c '.events[]'

EVENT_PATH=$(tempfile)

LOG_STREAM_NAME='2021/08/31/[$LATEST]6e531127fa5b4316a0d31da44aa84bfc'
aws logs get-log-events --log-group-name $LOG_GROUP_NAME --log-stream-name $LOG_STREAM_NAME | jq -c '.events[].message | select(test("^Message: ")) | ltrimstr("Message: ") | fromjson' > $EVENT_PATH


aws sqs list-queues | jq -c '.QueueUrls[] | select(test("System-Event-Processor-EventSQS"))'
QUEUE_URL="https://sqs.us-east-1.amazonaws.com/974862009434/System-Event-Processor-EventSQS-B4PSTUU6UQKT"
aws sqs send-message --queue-url $QUEUE_URL --message-body file://$EVENT_PATH


aws s3api list-buckets | jq -c '.Buckets[].Name'
BUCKET_NAME="salt"
OBJECT_KEY=$(aws s3api list-objects --bucket $BUCKET_NAME | jq -c '.Contents[].Key | select(test("0289045"))' | sed 's/"//g')

aws s3api head-object --bucket $BUCKET_NAME --key $OBJECT_KEY
aws s3api get-object --bucket $BUCKET_NAME --key $OBJECT_KEY /tmp/$(basename $OBJECT_KEY)

update_lambda_code.sh

set -euxo pipefail

TMPFILE=$(mktemp --suffix '.zip'); rm $TMPFILE
FILENAME=$(mktemp --suffix '.zip'); rm $FILENAME

LAMBDA="Group-Services-FindGroupTerminalsFunction-aca5Yk9reQma"
EVENT=$(cat << __JSON__ | jq -c
{
  "site_id": "0279927",
  "terminal_id": "a8:23:fe:58:58:e9",
  "correlation_id": "bd0142d6-cf4a-4d4e-866d-db789f014292"
}
__JSON__
)
DIRNAME=$HOME/dev//find-terminals-group
HANDLER="find_group.app.lambda_handler"
ECRIMG=public.ecr.aws/sam/build-python3.9:1.36-arm64

LAMBDA="Group-Services-GetRoomTerminalsFunction-gxKjF54A0yVB"
EVENT=$(cat << __JSON__ | jq -c
{
  "version": "0",
  "id": "c16a3a23-a292-cc82-0623-ea28e4a3e784",
  "detail-type": "cloud.room.checkin.v1",
  "source": "/app-api",
  "account": "952956287553",
  "time": "2022-01-14T22:16:53Z",
  "region": "us-east-1",
  "resources": [],
  "detail": {
    "site_id": "0279927",
    "room_number": "0003",
    "correlation_id": "363f3281-f1e8-44e8-87cf-3af8c8c1af53"
  }
}
__JSON__
)
DIRNAME=$HOME/dev//get-room-terminals
HANDLER="app.lambda_handler"
ECRIMG=public.ecr.aws/sam/build-python3.9:1.36-arm64

cd $DIRNAME

# NOTE: Run in arm containers so dependencies are correct
poetry export --format requirements.txt --output src/requirements.txt --without-hashes

cd src/

aws lambda update-function-configuration --function-name $LAMBDA --handler $HANDLER --layers "arn:aws:lambda:us-east-1:017000801446:layer:AWSLambdaPowertoolsPython:4" --environment "Variables={DB_SECRET_NAME=/site_api_read_write,DB_PROXY_ENDPOINT=proxy.proxy-cx6atckrdwx5.us-east-1.rds.amazonaws.com}" | jq -r .LastUpdateStatus
sleep 25

# docker run --rm -it \
#   -v ${PWD}:/var/task:Z -w /var/task \
#   $ECRIMG \
#   /bin/sh -c "pip install --upgrade -r requirements.txt --target ."

zip -r $FILENAME ./*

aws lambda update-function-code --function-name $LAMBDA --zip-file "fileb://$FILENAME" --publish | jq -r .LastUpdateStatus
sleep 5
aws lambda invoke  --function-name $LAMBDA --invocation-type RequestResponse --log-type Tail --payload "$(echo $EVENT | base64)" $TMPFILE | jq -r .LogResult | base64 --decode

upload_snc_lambda_function.sh

#!/usr/bin/env bash

tmpfile=$(mktemp /tmp/my-function-XXXX.zip)
rm $tmpfile

CODE_LOC=/home/jbrinkma/dev/
cd $CODE_LOC || exit 1

LAMBDA_NAME=$(aws lambda list-functions \
    | jq -r '.Functions[].FunctionName | select(test("SNC"))')

pip install --upgrade -r requirements.txt --target .
zip $tmpfile -r ./* &> /dev/null

RESULT=$(aws lambda update-function-code \
    --function-name $LAMBDA_NAME \
    --zip-file "fileb://$tmpfile" \
    --publish)

echo "$RESULT" | jq -r .LastUpdateStatus

# aws lambda update-function-configuration \
#     --function-name $LAMBDA_NAME \
#     --timeout 30 \
#     --memory-size 256 | jq -r .LastUpdateStatus

rm $tmpfile

verify_email_address.sh

#!/usr/bin/env bash

# Verify an email address so it can recieve email from our sandbox SES

EMAILADDR=${1}

aws ses verify-email-identity --email-address ${EMAILADDR}