Logrotate & upload nginx logfiles to s3://bucket/dt=..../hostname.log...gz

crontab

# Nginx - logrotate & upload to S3
0 0 * * * /usr/sbin/logrotate /etc/logrotate.d/nginx 
15 0 * * * s3cmd put /var/log/nginx/access.log-`date +"\%Y\%m\%d"`.gz s3://$LOGS_BUCKET_NAME/nginx-access/`date +"dt=\%Y\%m\%d"`/`hostname -s`.access.log-`date +"\%Y\%m\%d"`.gz

etc_logrotate.d_nginx

/var/log/nginx/error.log {
 daily
 rotate 30
 copytruncate
 compress
 notifempty
 missingok
}

/var/log/nginx/access.log {
 daily
 rotate 30
 compress
 missingok
 notifempty
 sharedscripts
 dateext
 postrotate
        kill -USR1 `\/usr\/bin\/pgrep -f \/usr\/local\/nginx\/sbin\/nginx` \
 endscript
}

upload-to-s3.sh

find /var/log/nginx/ -type f -mmin -10 -name "access.log*.gz" -exec sh -c "s3cmd put {} s3://$LOGS_BUCKET_NAME/nginx-access/\`date +\"dt=%Y%m%d\"\`/\`hostname -s\`.\`basename {}\`" \;

/home/domain1/upload_accesslog_to_s3.php

<?php
$cmd = 'find /var/log/nginx/domains -type f -mmin -50 -name "*.log*.gz" | grep -v error';
$output = shell_exec($cmd);
$file_array = preg_split('/\s+/', trim($output));

$count = 0;
foreach ($file_array as $file) {
        $count++;
        echo "File: ". $file ."\n";
        if (strpos($file, 'domain2') !== false) {
                echo "domain2: ";
                $milliseconds = round(microtime(true) * 1000);
                $cmd = "s3cmd put ". $file ." s3://domain2-access-logs/". $milliseconds . $count ."_domain2.gz";
                $output = shell_exec($cmd);
                echo $output ."\n";

                $cmd = "rm -rf $file";
                $output = shell_exec($cmd);
                echo $output ."\n";
        }
        elseif (strpos($file, 'domain1') !== false) {
                echo "domain1: ";
                $milliseconds = round(microtime(true) * 1000);
                $cmd = "s3cmd put ". $file ." s3://domain1-access-logs/". $milliseconds . $count ."_domain1.gz";
                $output = shell_exec($cmd);
                echo $output ."\n";

                $cmd = "rm -rf $file";
                $output = shell_exec($cmd);
                echo $output ."\n";
        }
        else {
                echo "No match for file, skipping.\n";
        }
}

/etc/logrotate.d/nginx

/var/log/nginx/domains/domain2.com.log {
 # hourly
 rotate 24
 compress
 missingok
 notifempty
}

/var/log/nginx/domains/domain2.com.error.log {
 # hourly
 rotate 24
 copytruncate
 compress
 notifempty
 missingok
}

/var/log/nginx/domains/domain1.net.error.log {
 # hourly
 rotate 24
 copytruncate
 compress
 notifempty
 missingok
}

/var/log/nginx/domains/domain1.net.log {
 # hourly
 rotate 24
 compress
 missingok
 notifempty
 sharedscripts
 postrotate
        kill -USR1 `\/usr\/bin\/pgrep -f \/usr\/local\/nginx\/sbin\/nginx` \
 endscript
}