View delete-orphan-s3-objects-in-production.sh
aws s3api list-objects-v2 --bucket govuk-assets-production --query 'Contents[?length(@.Key) == `24`].[Key]' --output text > orphan-s3-object-keys.txt
cat orphan-s3-object-keys.txt | wc -l
# ensure the result is 1137
cat orphan-s3-object-keys.txt | xargs -I {} aws s3api delete-object --bucket govuk-assets-production --key {} --output text > deleted-s3-objects.txt
aws s3api delete-object --bucket govuk-assets-production --key 9883ea2d-995f-4418-9ca1-abf31b572fa4 --output text >> deleted-s3-objects.txt
aws s3api delete-object --bucket govuk-assets-production --key be8a0003-4b25-4047-8a5f-f7ddc3c52716 --output text >> deleted-s3-objects.txt
View delete-orphan-s3-objects.sh
$ aws s3api list-objects-v2 --bucket govuk-assets-integration --query 'Contents[?length(@.Key) == `24`].[Key]' --output text >orphan-s3-object-keys.txt
$ cat orphan-s3-object-keys.txt | wc -l # => 1137
$ cat orphan-s3-object-keys.txt | xargs -I {} aws s3api delete-object --bucket govuk-assets-integration --key {}
View replicate-asset-manager-data.txt
local$ ./replicate-data-local.sh -n -p -q -e -t
dev-vm$ ./replicate-data-local.sh -s -p -q -e -t -i 'admin gov_delivery govuk_content imminence licence_finder manuals_publisher maslow panopticon publisher reviewomatic share_sale_publisher short_url_manager specialist_publisher_rebuild specialist_publisher_rebuild_standalone test travel_advice_publisher'
View list-unique-key-lengths-in-s3-bucket.sh
$ aws s3api list-objects-v2 --bucket <bucket-name> --query 'Contents[]' | jq 'map(.Key) | map(length) | unique'
[
24,
36
]
View find-s3-objects-with-specific-key-lengths.sh
$ aws s3api list-objects-v2 --bucket <bucket-name> --query 'length(Contents[?length(@.Key) == `24`])'
1137
$ aws s3api list-objects-v2 --bucket <bucket-name> --query 'length(Contents[?length(@.Key) == `36`])'
63950
$ aws s3api list-objects-v2 --bucket <bucket-name> --query 'length(Contents)'
65087
View count-objects-in-s3-bucket.sh
$ time aws s3api list-objects-v2 --bucket <bucket-name> --query 'length(Contents)'
65087
real 0m16.809s
user 0m2.808s
sys 0m0.421s
$ time aws s3 ls s3://<bucket-name> | wc -l
65087
View cloudwatch-bucket-size-bytes-over-time.sh
aws --region eu-west-1 --profile production cloudwatch get-metric-statistics --namespace AWS/S3 --metric-name BucketSizeBytes --dimensions Name=BucketName,Value=jm-govuk-assets-production Name=StorageType,Value=StandardStorage --start-time 2017-11-01T00:00 --end-time 2017-11-18T00:00 --period 3600 --statistic Maximum | jq -r '.Datapoints[] | [.Timestamp, .Maximum] | @csv' | sort
"2017-11-07T00:00:00Z",1365
"2017-11-08T00:00:00Z",734821
"2017-11-09T00:00:00Z",64893805
"2017-11-10T00:00:00Z",64893805
"2017-11-11T00:00:00Z",64893805
"2017-11-12T00:00:00Z",64893805
"2017-11-13T00:00:00Z",64893805
"2017-11-14T00:00:00Z",64893805
"2017-11-15T00:00:00Z",64893805
View 2017-11-06-gfr-gds-tech-lead-notes.md

Achievements

  • Whitehall now creates, updates & deletes organisation logos in Asset Manager
  • Existing Whitehall organisation logos have been moved to Asset Manager
  • Whitehall organisation logos are being served by Asset Manager
  • Manually triggered replication for production assets on S3 which pre-dated replication being enabled

Ongoing and next

  • Stop saving Whitehall organisation logo assets to Whitehall bit of NFS mount
View saving-for-old-age-rule-of-thumb.md

By the time you're 30, aim to have 1x your annual income set aside for retirement. At 40, 3x; at 50, 6x; at 60, 8x; and by retirement, 10x.

View 2017-10-30-gds-gfr-update.md

Achievements

  • Replication set up on Asset Manager S3 production bucket to provide backup
  • Asset Manager uses a fake S3 service in development by default

Ongoing and next

  • Change Whitehall to create/update/delete new organisation logo assets in Asset Manager
  • Move existing Whitehall organisation logos assets to Asset Manager
  • Serve Whitehall organisation logo assets from Asset Manager