Bash script to Create tar archives by year and month. eg: 201710.tar.gz/201703.tar.gz

tarbyYearAndMonth.sh

#!/bin/bash
# Script to tar all files within a given source directory aggregated by year and month
# generates files like:
# 201701.tar.gz 201805.tar.gz etc...


# Check to see if parallel command & pigz command is available
command -v parallel >/dev/null 2>&1 || { echo >&2 "I require Parallel but it's not installed.  Aborting."; exit 1; }
command -v pigz >/dev/null 2>&1 || { echo >&2 "I require Pigz but it's not installed.  Aborting."; exit 1; }

Fhelp(){
echo "$(basename $0) [full/cron] <sourcedir> <targetdir>"
echo ""
echo "full:  Backup all files to tgz. Existing Backupfiles"
echo "       will not be overwritten"
echo "cron:  Backup all files to tgz and update possibly"
echo "       existing Backupfiles"
}

sourcedir="files"
targetdir="backup"
tmpdir="/tmp"

# How many threads should pigz & parallel use
threads="8"

# Add/change lines to exclude more folders
exlcude_list="--exclude=files/testfolder"
exclude_list="$exclude_list --exclude=files/testfolder2/*"

# Use the directory we get from parameter interactively
case "$1" in
	full)
		type=full
	;;
	cron)
		type=cron
	;;
	*)
		Fhelp
		exit 0
	;;
esac
test -z "$2" || sourcedir="$2"
test -z "$3" || targetdir="$3"

echo "This is what I'm going to do: TAR IN $1 FROM $sourcedir TO $targetdir" 

sourcedirname=$(basename "$sourcedir")

rm -rf "$tmpdir/"*".tarbydate.tmp" &> /dev/null

echo "Create File List"
find "$sourcedir" -type f -printf '\n%TY%Tm %p' | parallel -j+"$threads" --no-run-if-empty --colsep ' ' echo {2} ">>" "$tmpdir/${sourcedirname}_"{1}.tarbydate.tmp

echo "Create Backup Files"
ls "$tmpdir/"*".tarbydate.tmp" | while read line
do
	backupfilename="$(basename $line | cut -d "." -f 1).tgz"
	echo "#-- ${backupfilename}"
	case "$type" in
		full)
			if test -f "$targetdir/$backupfilename"
			then
				echo "- Skipping - already exists!"
				continue
			fi
			if ! tar cpf - ${exclude_list} -T "$line" 2> /dev/null | pigz -p "$threads" > "$targetdir/$backupfilename" 2> /dev/null
			then
				echo "- Failed with ERROR"
				continue
			else
				echo "- Created successfully!"
			fi
		;;
		cron)
			if test -f "$targetdir/$backupfilename"
			then
				echo "- Decompressing.."
				gunzip "$targetdir/$backupfilename" 2> /dev/null
				echo "- Updating.."
				if ! tar uvf "$targetdir/${backupfilename%????}.tar.gz" ${exclude_list} -T "$line" &> /dev/null
				then
					echo "- Failed with ERROR"
					continue
				else
					cat "$targetdir/${backupfilename%????}.tar.gz" | pigz -p "$threads" > "$targetdir/$backupfilename" 2> /dev/null
					rm "$targetdir/${backupfilename%????}.tar.gz" &> /dev/null
					echo "- Updated successfully!"
				fi
			else
				if ! tar cpf - ${exclude_list} -T "$line" 2> /dev/null | pigz -p "$threads" > "$targetdir/$backupfilename" 2> /dev/null
				then
					echo "- Failed with ERROR"
					continue
				else
					echo "- Created successfully!"
				fi
			fi
		;;
	esac
done

rm -rf "$tmpdir/"*".tarbydate.tmp" &> /dev/null