clean_old_files.sh

#!/usr/bin/env python

# Clean up old files around the server, called weekly Sunday 8am
# mReschke 2012-05-01

import os
import sys
import subprocess
import traceback
import datetime


def clean_oldfiles():

	# ImportControl Processed 30days
	clean('/store/data/Production/ftp/importcontrol/DYNA-SQL/processed/',  30)
	clean('/store/data/Production/ftp/importcontrol/DYNA-SQL2/processed/', 30)
	clean('/store/data/Production/ftp/importcontrol/DYNA-SQL3/processed/', 30)
	clean('/store/data/Production/ftp/importcontrol/DYNA-SQL4/processed/', 30)
	clean('/store/data/Production/ftp/importcontrol/DYNA-SQL5/processed/', 30)
	clean('/store/data/Production/ftp/importcontrol/DYNA-SQL6/processed/', 30)
	clean('/store/data/Production/ftp/importcontrol/DYNA-SQL7/processed/', 30)

	# WebAQ
	clean('/store/data/Development/data/WebAQ/VCP/Attachment/', 21)
	clean('/store/data/Production/data/WebAQ/VCP/Attachment/',  21)
	clean('/store/data/Development/data/WebAQ/tmpFiles/',       21)
	clean('/store/data/Production/data/WebAQ/tmpFiles/',        21)

	# ESP
	clean('/store/data/Production/data/ESP/ESPReport1/Archives/', 540)
	# cannot clean from ESP2 without removing from DB too, so not here

	# DynaMail
	clean('/store/data/Production/data/DynaMail/DBA_Reports/',    30)
	clean('/store/data/Production/data/DynaMail/Disclaimers/',    30)
	clean('/store/data/Production/data/DynaMail/LiveCallOrders/', 30)
	clean('/store/data/Production/data/DynaMail/PA_Reports/',     30)
	clean('/store/data/Production/data/DynaMail/Proposals/',      30)
	clean('/store/data/Production/data/DynaMail/SalesOrders/',    30)

	# DynaMailPDF
	clean('/store/data/Production/data/DynaMail/DynaMailPDF', 720, '*.txt',  2)
	clean('/store/data/Production/data/DynaMail/DynaMailPDF', 720, '*.csv',  2)
	clean('/store/data/Production/data/DynaMail/DynaMailPDF', 720, '*.html', 2)
	clean('/store/data/Production/data/DynaMail/DynaMailPDF', 720, '*.pdf',  3)

	# NCOA Archives
	clean('/store/data/Production/data/NCOA', 720)

	# Temp Folders
	clean('/store/sys/tmp/', 5)
	clean('/store/data/Production/tmp/', 30)
	clean('/store/data/Production/tmp/DealerMenus', 5)

	# Logs
	clean('/store/data/Production/log/', 540) #Entire log folder generic 901 days
	clean('/store/data/Production/ftp/CarMailPrintReview/GenLogs/', 30)
	clean('/store/data/Production/log/DynaMailGeneratorTestList/', 180)
	clean('/store/data/Production/log/DynaMailGenerator/', 360)
	clean('/store/data/Production/log/LogCompiler/', 30)
	clean('/store/data/Production/log/ImportControl/', 60)
	clean('/store/data/Production/log/Xenlb/haproxy/', 30)
	clean('/store/data/Production/log/Xenstore/Proftpd/', 30)
	clean('/store/data/Production/log/Xenstore/CleanOldFiles/', 30)
	clean('/store/data/Production/log/ICGenerator/', 90)
	clean('/store/data/Production/log/DMSPush/', 90)

	# FTP
	clean('/store/data/Production/ftp/', 30, '*-deleted', 1, True) #cleaned deleted account folders
	clean('/store/data/Production/ftp/rci/processed/', 7)
	clean('/store/data/Production/ftp/parkwayford/', 30, '*.csv')
	clean('/store/data/Production/ftp/parkwayford/', 30, '*.pdf')
	clean('/store/data/Production/ftp/qmalouann/', 30, '*.*', 999, True)
	clean('/store/data/Production/ftp/CarMailPrintReview/OLD-CARMAIL-FILES/', 30, '*.*', 999, True)
	clean('/store/data/Production/ftp/bmwdeclines/', 60)

	# DMS Push (there are millions, purge daily or so)
	clean('/store/data/Production/data/DMSPush/', 2)
	# DMSPush is obsolete now

	# VFIOpenImport
	clean('/store/data/Production/data/VFIOpenImport/', 7)

	# DynaComm
	clean('/store/data/Production/data/DynaComm/', 7)
	clean('/store/data/Development/data/DynaComm/', 7)

	# Parser Rsync
	clean('/store/data/Production/data/Parser/', 30)

	# Other
	cmd('cat /dev/null > /var/log/proftpd/tls.log')
	cmd('cat /dev/null > /var/mail/mail')


################################################################################
################################################################################
def cmd(run, capture_output=False):
    # Run the cmd
	if capture_output:
		proc = subprocess.Popen(run, universal_newlines=True, executable='bash', shell=True, stdout=subprocess.PIPE).stdout
		return proc.read().strip()
	else:
		run = "bash -c '" + run + "'"
		os.system(run)
		#print run

def clean(path, older_than_days, file_filter='*', max_depth=999, include_folders=False):
	del_cmd = 'rm -rfv'
	#del_cmd = 'echo'
	typestr = '-type f'
	if include_folders: typestr = ''
	cmd('find ' + path + ' -maxdepth ' + str(max_depth) + ' -iname "' + file_filter + '" ' + typestr + ' -mtime +' + str(older_than_days) + ' -exec ' + del_cmd + ' {} \; | tee -a ' + log)

def finish():
	cmd('touch /tmp/clean_oldfiles.end.alive')
	exit(0)
################################################################################
try:
	# Globals
	now = datetime.datetime.now()
	log='/store/data/Production/log/Xenstore/CleanOldFiles/' + now.strftime("%Y-%m-%d") + '_CleanOldFiles.log'

	cmd('touch /tmp/clean_oldfiles.begin.alive')
	clean_oldfiles()
	finish()
except Exception as e:
    # On any error return 0 which means success.
    # If this script returns 1 then postfix thinks mail delivery failed so it will
    # Bounce back to the sender which would contain the error details like script line number, script
    # On any errors, just return success so no mail will ever be returned
	print str(e) + traceback.format_exc()
	finish()
################################################################################