troubleshooting tools

tips_and_tricks.sh

sysdig -s 2000 -A -c echo_fds proc.name=chef-client and fd.type=ipv4


chef-shell> node.run_list.expand(node.chef_environment).run_list_trace.each { |k,v| puts "#### " + k.to_s + "\n"+ v.join("\n") if v.grep(/___PATTERN___/).any?  }

# Disable the stupid auto-logout (https://stackoverflow.com/questions/17397069/unset-readonly-variable-in-bash/21294582#21294582)
unset TMOUT &> /dev/null
if [[ $? -ne 0 ]]; then
  comm=$(echo -e "attach $$\ncall unbind_variable(\"TMOUT\")\ndetach\nquit")
  gdb="gdb -q -n"
  [[ 1 -eq "$(cat /proc/sys/kernel/yama/ptrace_scope)" ]] && gdb="sudo $gdb"
  $gdb <<< "$comm"
fi
echo $TMOUT

# replace "%{hiera('some_param')}" with "%{alias('some_param')}"
find ./hieradata -name '*.yaml' | while read file; do sed -i "s,\"%{hiera('\([^']\+\)')}\",\"%{alias('\1')}\",g" $file; done

# run atop
docker run --rm -ti --pid host alpine:edge sh -c 'apk add --update --no-cache atop && exec atop 1'

# debug container
docker run -v /var/run/docker.sock:/var/run/docker.sock --rm -ti --pid host --net host alpine:edge sh -c 'apk add --update --no-cache bash netcat-openbsd atop tcpdump nmap bind-tools docker tshark && TERM=screen exec bash'

# HTTP transaction timing breakdown 
while :; do curl -H "Accept-Encoding: gzip" -s -o /dev/null -w "$(date) %{time_namelookup} %{time_connect} %{time_appconnect} %{time_pretransfer} %{time_redirect} %{time_starttransfer} %{time_total} %{size_download}\n" https://google.com; done

# Show all PIDs that listen 4300 TCP port
lsof -iTCP:4300 -sTCP:LISTEN -a -nP

# show HTTP request and response
tshark -nn -i eno1 -Y "http.request or http.response" 'host 1.2.3.4 and tcp port 80'

# shows all possible jq paths https://github.com/stedolan/jq/issues/243
# Example output:
# "."
# ".name"
# ".normal"
# ".normal.policy_group"
# ".normal.policy_name"
# ".normal.redisio"
# ".normal.redisio.servers"
# ".normal.redisio.servers[]"
# ".normal.redisio.servers[].backuptype"
# ".normal.redisio.servers[].breadcrumb"
❯ knife node show some.server.com -F json | jq '[path(..)|map(if type=="number" then "[]" else tostring end)|join(".")|split(".[]")|join("[]")]|unique|map("."+.)|.[]'

# jq - recursive search for keys containing "string" stripping empty results
❯ knife node show some.server.com -l -F json | jq '.. | objects | with_entries(select(.key | contains("chef_"))) | select(. != {})'
{
  "chef_packages": {
    "chef": {
      "version": "12.21.31",
      "chef_root": "/opt/chef/embedded/lib/ruby/gems/2.3.0/gems/chef-12.21.31/lib"
    },
    "ohai": {
      "version": "8.25.1",
      "ohai_root": "/opt/chef/embedded/lib/ruby/gems/2.3.0/gems/ohai-8.25.1/lib/ohai"
    }
  }
}

#
❯ knife node show some.server.com -l -F json | jq -r '.. | objects | .chef_packages.chef.version | select(length > 0)'
12.21.31


### Match SSH Keys
cat public_key ssh-keygen -f /dev/stdin -e -m PEM | openssl rsa -RSAPublicKey_in -modulus -noout
cat private_key openssl rsa -modulus -noout

### RabbitMQ
# Show queues status
VHOST=/
rabbitmqctl list_queues -p $VHOST \
  name \
  durable \
  auto_delete \
  policy \
  pid \
  slave_pids \
  synchronised_slave_pids \
  state \
  messages \
  | column -t
# Show 10 messages in a queue
VHOST=/
USER=root
PASS=somepass
curl -s http://127.0.0.1:15672/cli/rabbitmqadmin | python - -u $USER -p $PASS -V $VHOST -f long get queue=ApiRequestStats count=10 | less -S

### AWS
# Set temporary keys for current session
eval $(
  aws sts assume-role \
  --role-arn arn:aws:iam::$$$123456$$$$:role/OrganizationAccountAccessRole \
  --role-session-name "RoleSession1" \
  --profile $$$ROOT$$$ \
  | jq -r '.Credentials | "export AWS_ACCESS_KEY_ID=\(.AccessKeyId) AWS_SECRET_ACCESS_KEY=\(.SecretAccessKey) AWS_SESSION_TOKEN=\(.SessionToken)"'
 )

# Unset params
unset AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN