Skip to content

Instantly share code, notes, and snippets.

@kgorskowski
kgorskowski / kafka_sizing.py
Created July 19, 2023 13:11
kafka broker sizing calculator based on available storage
def calculate_retention_settings(storage_per_broker, brokers, partitions, replication, data_rate):
# Calculate total available storage in the cluster
# consider the lifecycle management cycle as overhead (runs per default every 5 minutes)
lc_overhead = data_rate * 1024 * 1024 * 60 * 5
# total storage in GB
total_storage = storage_per_broker * brokers
# total storage in bytes
total_storage_bytes = total_storage * (1024*1024*1024)
# usable bytes minus 20% and lifecycle overhead
#cloud-config
# Ensure that walinuxagent starts after cloud-init finished. That prevents the DevOps Agent starting before the Docker Daemon is ready and logged in to jfrog
bootcmd:
- mkdir -p /etc/systemd/system/walinuxagent.service.d
- echo "[Unit]\nAfter=cloud-final.service" > /etc/systemd/system/walinuxagent.service.d/override.conf
- sed "s/After=multi-user.target//g" /lib/systemd/system/cloud-final.service > /etc/systemd/system/cloud-final.service
- systemctl daemon-reload
groups:
@kgorskowski
kgorskowski / treesize.sh
Created December 16, 2020 13:42
treesize alternative for linux shell
#/bin/sh
du -k --max-depth=1 | sort -nr | awk '
BEGIN {
split("KB,MB,GB,TB", Units, ",");
}
{
u = 1;
while ($1 >= 1024) {
$1 = $1 / 1024;
u += 1
@kgorskowski
kgorskowski / template.sh
Created December 15, 2020 15:13
copy of minimal secure bash script layout
#!/usr/bin/env bash
set -Eeuo pipefail
cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1
trap cleanup SIGINT SIGTERM ERR EXIT
usage() {
cat <<EOF
@kgorskowski
kgorskowski / packer-template.json
Last active September 26, 2018 10:52
template for my default packer/chef-solo workflow
{
"builders": [{
"ami_block_device_mappings": [{
"device_name": "/dev/sdb",
"no_device": "true"
},
{
"device_name": "/dev/sdc",
"no_device": "true"
}
flist=(`aws s3 ls s3://karsten-lambda-test --region eu-central-1 | awk '{print $4}'`)
for i in $flist; do aws s3 cp s3://karsten-lambda-test/$i . --region eu-central-1; guetzli --quality 90 $i $i.jpeg; mv $i.jpeg $i; aws s3 cp $i s3://karsten-lambda-test/$i --region eu-central-1; aws s3api put-object-tagging --bucket karsten-lambda-test --key $i --tagging "TagSet=[{Key=guetzli,Value=true}]" --region eu-central-1; done
@kgorskowski
kgorskowski / create_deployment.sh
Last active October 31, 2018 22:45
recreate latest successful codedeploy deployment. needs awscli, jq and corresponding iam permissions
#!/bin/bash
#your variables here, you can set AWS credentials here as well
AWS_REGION=$YourAWSRegion
DEP_GROUP=$NameOfYourCodeDeployDeploymentGroup
APP_NAME=$NameOfYourCDApplication
# list successfull deployments only
STATUS=Succeeded
@kgorskowski
kgorskowski / associate_ip.sh
Created January 27, 2017 11:33
Reassociate AWS EIP (needs awscli, jq and the corresponding iam permissions)
#!/bin/bash
# Helper Functions
get_instance_region() {
if [ -z "$AWS_REGION" ]; then
AWS_REGION=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document \
| grep -i region \
| awk -F\" '{print $4}')
fi
echo $AWS_REGION
#!/bin/bash
ADMIN_ACC_ID=$1
SESSION_NAME="adminsession"
case "$1" in
'Account1')
ADMIN_ACC_ID=1234567890
AWS_REGION=eu-central-1
;;
'Account2')
#!/bin/bash
hostname="consul-$(curl -s http://169.254.169.254/latest/meta-data/instance-id)"
CONF=/etc/consul/config/000-consul.json
TEMPLATE="/mnt/dashboard/consul-config/config.json"
x=0
rm -f $CONF
rm -f /tmp/instances $$ rm -f /tmp/*.json
restart consul