fduran

Troubleshooting

Intro

The incident management steps I have in mind when being on-call and getting an alert are:

• Verify the issue
• Triage
• Communicate and scalate if needed
• Mitigate

fduran

#!/bin/bash
# generate a number of files with random sizes in a range
 
min=1      # min size (MB)
max=10     # max size (MB)
nofiles=20 # number of files
 
for i in `eval echo {1..$nofiles}`
do
    dd bs=1M count=$(($RANDOM%max + $min)) if=/dev/urandom of=./files/file$i

fduran

# www.fduran.com
# Create Java KeyStore from SSL certificate for domain example.com

# Change format from cert.crt PEM (----BEGIN CERTIFICATE----- ...  -----END CERTIFICATE-----) to DER
openssl x509 -in cert.crt -inform PEM -out cert.der -outform DER

# create KeyStore cert.jks for your domain ("alias" in keytool, "common name" or CN in openssl req)
keytool -import -trustcacerts -alias example.com -file cert.der -keystore cert.jks

fduran

# www.fduran.com
# redirect from apache port (:8080 for ex for tomcat etc) to subdomain
 
# in apache config:
<VirtualHost *:80>
    ServerName subdomain.example.com
    ProxyPass / http://127.0.0.1:8080/
    ProxyPassReverse / http://127.0.0.1:8080/
</VirtualHost>

fduran

# www.fduran.com

# disk space usage
df -h

# biggest (10) directories in size (MB)
du -mxS / | sort -n | tail -10

# clean package cache
apt-get clean

fduran

# better command line history: more commands kept, aggregate, add timestamp
# www.fduran.com

# add in /etc/profile

shopt -s histappend
HISTSIZE=10000
HISTTIMEFORMAT="%Y-%m-%d %H:%M:%S "
HISTCONTROL=ignoredups
PROMPT_COMMAND="history -a;history -c;history -r;$PROMPT_COMMAND"

fduran

#!/bin/bash
# www.fduran.com
# script that will send an email to EMAIL when disk use in partition PART is bigger than %MAX
# adapt these 3 parameters to your case
MAX=95
EMAIL=alert@example.com
PART=sda1

USE=`df -h |grep $PART | awk '{ print $5 }' | cut -d'%' -f1`
if [ $USE -gt $MAX ]; then

fduran

// www.fduran.com
// send mail with Amazon AWS SES without extra library
// needs Pear: apt-get install php-pear 
<?php
function mailSES($to, $subject, $body, $from)
{
        include_once('Mail.php');

	$headers["From"] = $from;
	$headers["To"] = $to;

fduran

# chkrootkit & rkhunter:
# www.fduran.com

http://www.chkrootkit.org/download/
http://sourceforge.net/projects/rkhunter/
cd /usr/local/src/
wget ftp://ftp.pangeia.com.br/pub/seg/pac/chkrootkit.tar.gz
tar zxvf chkrootkit.tar.gz
cd chkrootkit-0.49/
./chkrootkit

fduran

# www.fduran.com
# hardware stress test
# mismatched md5sums shows a faulty disk/RAM  
# time depends on hardware, ex: 1 sec per 100 count
dd if=/dev/zero of=/tmp/test.file bs=1M count=1000
for i in {1..5}; do md5sum /tmp/test.file; done