set hive.execution.engine=tez;
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
############################################################################################################################################################ | |
# Cloudtrail recursively search through all events in different folders relating to a specific log group which generated # | |
############################################################################################################################################################ | |
find . -name '*.json' -exec cat {} \; | jq '.Records[] | select(.requestParameters.logGroupName=="/mnt/log/communications-delivery-stage")' | jq -s '[ .[] | select(.errorCode=="ResourceAlreadyExistsException") ] | unique_by(.eventName)' | |
find . -name '*.json' -exec cat {} \; | jq '.Records[] | select(.requestID=="3ddb4d1f-41d2-11e8-8533-1dadb66cbff4")' | |
-- Count number of Exceptions | |
find . -name '*.json' -exec cat {} \; | jq '.Records[]' | jq -s '[ .[] | select(.errorCode=="ResourceAlreadyExistsException") ] | length' |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/bash | |
############################################################################## | |
# assume role and populate in environment variables # | |
############################################################################## | |
#using meta data | |
url="http://169.254.169.254/latest/meta-data/" | |
iam="iam/security-credentials/" | |
security_credentials=$(curl -s "$url$iam") |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package main | |
import ( | |
"context" | |
"fmt" | |
"time" | |
"github.com/aws/aws-sdk-go/aws" | |
"github.com/aws/aws-sdk-go/aws/session" | |
"github.com/aws/aws-sdk-go/aws/client" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/bash | |
a=0 | |
while [ $a -lt 20 ] | |
do | |
now=$(TZ='UTC' date +%s%3N) | |
message=$(cat << EndOfMessage | |
{ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import json | |
import sys | |
import boto3 | |
import time | |
## boto3 client | |
logs = boto3.client('logs', region_name='eu-west-1') | |
## Define log group and log stream. Default log group for glue logs and logstream defined dynamically upon job run. | |
log_group = "my_custom_log_group" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env bash | |
# metric math alarms | |
aws cloudwatch put-metric-alarm --alarm-name test2 --evaluation-periods 2 --alarm-actions arn:aws:sns:eu-west-1:037559324442:cloudwatch-sqs --threshold 1 --comparison-operator GreaterThanThreshold --metrics '[{"Id":"e1","Label":"Expression1","ReturnData":true,"Expression":"SUM(METRICS()/2)"},{"Id":"m1","ReturnData":false,"MetricStat":{"Metric":{"MetricName":"Available","Dimensions":[{"Name":"WorkspaceId","Value":"ws-fdw4f2pt8"}],"Namespace":"AWS/WorkSpaces"},"Period":300,"Stat":"Average"}}]' | |
metrics=$(cat ~/temp/temp2.json | | |
#temp=$(aws cloudwatch list-metrics --metric-name CPUUtilization --namespace AWS/EC2 --region eu-west-1 --query "Metrics[?Dimensions[0].Name == 'InstanceId']" | | |
# when using timestamp as an ID - now | tostring | split(".")[1] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Gets instances without labels in the widgets | |
labelLessInstance=$( cat temp.json | | |
jq ' | |
. | | |
( | |
[ | |
.widgets[] | | |
select( | |
(.type == "metric") and (.properties.metrics[0][0] == "AWS/EC2") | |
) | |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env bash | |
query_string=$(cat << EndOfMessage | |
fields @timestamp, @logStream, headers.X-Amzn-Trace-Id, @transId, @message | |
| parse @message /(transactionId:[ ]?)(?<@transId>[a-zA-Z0-9]+)/ | |
| filter @transId = a4c475516be5445a87fbb81bb7a4b365 | |
EndOfMessage | |
) \ | |
&& \ | |
query_id=`aws logs start-query --log-group-name /aws/lambda/console_log \ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import com.amazonaws.auth.AWSStaticCredentialsProvider; | |
import com.amazonaws.auth.BasicSessionCredentials; | |
import com.amazonaws.services.costexplorer.AWSCostExplorer; | |
import com.amazonaws.services.costexplorer.AWSCostExplorerClientBuilder; | |
import com.amazonaws.services.costexplorer.model.*; | |
import com.amazonaws.services.securitytoken.AWSSecurityTokenService; | |
import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClientBuilder; | |
import com.amazonaws.services.securitytoken.model.AssumeRoleRequest; | |
import com.amazonaws.services.securitytoken.model.Credentials; |
OlderNewer