Simple test of BackupPC_dump subroutines: BackupExpire and BackupFullExpire.

BackupExpire-test.pl

#!/usr/bin/perl
#============================================================= -*-perl-*-
# Simple test of BackupPC_dump subroutines:
# BackupExpire and BackupFullExpire.
#========================================================================
#
# AUTHOR
#   Alexander Moisseev <moiseev@mezonplus.ru>
#
# COPYRIGHT
#   Copyright (C) 2015 Alexander Moisseev
#   Copyright (C) 2001-2013  Craig Barratt
#
#   This program is free software: you can redistribute it and/or modify
#   it under the terms of the GNU General Public License as published by
#   the Free Software Foundation, either version 3 of the License, or
#   (at your option) any later version.
#
#   This program is distributed in the hope that it will be useful,
#   but WITHOUT ANY WARRANTY; without even the implied warranty of
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#   GNU General Public License for more details.
#
#   You should have received a copy of the GNU General Public License
#   along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
#========================================================================
# Version 2015-05-03
# BackupPC subs version 4.0.0alpha3, released 1 Dec 2013.
#========================================================================

use strict;
#use warnings;

use Data::Dumper;

#
# BackupPC configuration options
#

# Initial configuration
my %ConfOrig = (

    # if set to zero then fill/unfilled will match full/incremental
    FillCycle      => 0,
    FullPeriod     => 1,
    FullKeepCnt    => [ 128 ],
    FullKeepCntMin => 30,
    FullAgeMax     => 90,

    IncrKeepCnt    => 6,
    IncrKeepCntMin => 1,
    IncrAgeMax     => 1,
);

# Configuration will be changed to after $preConfChgNum backup saved
my %ConfNew = (

    # if set to zero then fill/unfilled will match full/incremental
    FillCycle      => 0,
    FullPeriod     => 1,
    FullKeepCnt    => [ 3, 0, 0, 15 ],
    FullKeepCntMin => 30,
    FullAgeMax     => 90,

    IncrKeepCnt    => 6,
    IncrKeepCntMin => 1,
    IncrAgeMax     => 1,
);

my $oldestNum      = 0;       # The oldest full backup number
my $preConfChgNum  = 126;     # Backup number prior to config change
my $newestNum      = 127;     # The latest backup number
my $backupDuration = 3600;    # Perform every expire check at backup start time + this value

my $startTime =
  time -
  ( $ConfOrig{FullPeriod} * ( $preConfChgNum - $oldestNum + 1 ) +
      $ConfNew{FullPeriod} * ( $newestNum - $preConfChgNum ) ) * 24 * 3600 -
  $backupDuration;

#
# Declarations for sub BackupExpire and sub BackupFullExpire
#
my $XferLOG;
my $TopDir = "nowhere";
my %opts = ( v => 1 );
*LOG = *STDOUT;

my @Backups;

$Data::Dumper::Indent   = 0;
$Data::Dumper::Sortkeys = 1;
$Data::Dumper::Terse    = 1;

my $bpc = main->new;

print "
 **********
 * Doing full backups and expire checks with %ConfOrig
 **********
";

my %Conf = %ConfOrig;
DoDumps( $oldestNum, $preConfChgNum );

print "
 **********
 * Configuration has changed to %ConfNew
 **********
";

%Conf = %ConfNew;
DoDumps( $preConfChgNum + 1, $newestNum );

##############
# Subroutines
##############

sub DoDumps {
    my ( $from, $to ) = @_;
    foreach my $num ( $from .. $to ) {
        print "\n ==> Make full backup #$num\n";
        BackupSave($num);
        BackupExpire("some_client");
        $num++;
    }
    #DumpArrayOfHashes(@Backups);
}

#
# Adds new backup to the test array
#
sub BackupSave {
    my $num = shift;

    $startTime += $Conf{FullPeriod} * 24 * 3600;
    push @Backups, {
        num       => $num,
        startTime => $startTime,
        type      => "full",
        noFill    => 0,
        version   => "",           # Empty means preV4
    };
}

sub DumpArrayOfHashes {
    foreach my $hash (@_) {
        print Dumper($hash), "\n";
    }
}

#
# Removes a specific backup
#
sub BackupRemove {
    my ( $client, $idx ) = @_;

    my $bkupNum = $Backups[$idx]{num};
    #print("__bpc_progress_state__ delete #$bkupNum\n");
    splice( @Backups, $idx, 1 );
    return 0;
}

sub BackupInfoRead {
    return @Backups;
}

sub new {
    my $class = shift;
    my $self  = {};
    bless $self, $class;
    return $self;
}

sub timeStamp {
    return "";
}

sub BackupInfoWrite {
    my ( $bpc, $client, @Backups ) = @_;
    for ( my $i = 0 ; $i < @Backups - 1 ; $i++ ) {
        my $timedelta =
          ( $Backups[ $i + 1 ]{startTime} - $Backups[$i]{startTime} ) / 86400;
        print $Backups[$i]{num}, "<", $timedelta, ">";
    }
    print $Backups[-1]{num}, "\n";
}

##############
# Unmodified BackupPC_dump subroutines (4.0.0alpha3)
##############

#
# Decide which old backups should be expired.
#
sub BackupExpire
{
    my($client) = @_;
    my($Dir) = "$TopDir/pc/$client";
    my($cntFull, $cntIncr, $firstFull, $firstIncr, $oldestIncr,
       $oldestFull, $changes);

    @Backups = $bpc->BackupInfoRead($client);
    if ( $Conf{FullKeepCnt} <= 0 ) {
        print(LOG $bpc->timeStamp,
                  "Invalid value for \$Conf{FullKeepCnt}=$Conf{FullKeepCnt}; not expiring any backups\n");
	print(STDERR
            "Invalid value for \$Conf{FullKeepCnt}=$Conf{FullKeepCnt}; not expiring any backups\n")
			    if ( $opts{v} );
        return;
    }
    while ( 1 ) {
	$cntFull = $cntIncr = 0;
	$oldestIncr = $oldestFull = 0;
	for ( my $i = 0 ; $i < @Backups ; $i++ ) {
            $Backups[$i]{preV4} = ($Backups[$i]{version} eq "" || $Backups[$i]{version} =~ /^[23]\./) ? 1 : 0;
            if ( $Backups[$i]{preV4} ) {
                if ( $Backups[$i]{type} eq "full" ) {
                    $firstFull = $i if ( $cntFull == 0 );
                    $cntFull++;
                } elsif ( $Backups[$i]{type} eq "incr" ) {
                    $firstIncr = $i if ( $cntIncr == 0 );
                    $cntIncr++;
                }
            } else {
                if ( !$Backups[$i]{noFill} ) {
                    $firstFull = $i if ( $cntFull == 0 );
                    $cntFull++;
                } else {
                    $firstIncr = $i if ( $cntIncr == 0 );
                    $cntIncr++;
                }
            }
	}
	$oldestIncr = (time - $Backups[$firstIncr]{startTime}) / (24 * 3600)
                        if ( $cntIncr > 0 );
	$oldestFull = (time - $Backups[$firstFull]{startTime}) / (24 * 3600)
                        if ( $cntFull > 0 );

        $XferLOG->write(\"BackupExpire: cntFull = $cntFull, cntIncr = $cntIncr, firstFull = $firstFull,"
                   . " firstIncr = $firstIncr, oldestIncr = $oldestIncr, oldestFull = $oldestFull\n")
                                        if ( $XferLOG );

        #
        # In <= 3.x, with multi-level incrementals, several of the
        # following incrementals might depend upon this one, so we
        # have to delete all of the them.  Figure out if that is
        # possible by counting the number of consecutive incrementals
        # that are unfilled and have a level higher than this one.
        #
        # In >= 4.x any backup can be deleted since the changes get
        # merged with the next older deltas, so we just do one at
        # a time.
        #
        my $cntIncrDel = 1;
        my $earliestIncr = $oldestIncr;

	for ( my $i = $firstIncr + 1 ; $i < @Backups ; $i++ ) {
            last if ( !$Backups[$i]{preV4} || $Backups[$i]{level} <= $Backups[$firstIncr]{level}
                          || !$Backups[$i]{noFill} );
            $cntIncrDel++;
            $earliestIncr = (time - $Backups[$i]{startTime}) / (24 * 3600);
        }

	if ( $cntIncr >= $Conf{IncrKeepCnt} + $cntIncrDel
		|| ($cntIncr >= $Conf{IncrKeepCntMin} + $cntIncrDel
		    && $earliestIncr > $Conf{IncrAgeMax}) ) {
            #
            # Only delete an incr backup if the Conf settings are satisfied
            # for all $cntIncrDel incrementals.  Since BackupRemove() updates
            # the @Backups array we need to do the deletes in the reverse order.
            # 
            for ( my $i = $firstIncr + $cntIncrDel - 1 ;
                    $i >= $firstIncr ; $i-- ) {
                print("removing unfilled backup $Backups[$i]{num}\n");
                $XferLOG->write(\"removing unfilled backup $Backups[$i]{num}\n") if ( $XferLOG );
                last if ( BackupRemove($client, $i, 1) );
                $changes++;
            }
            next;
        }

        #
        # Delete any old full backups, according to $Conf{FullKeepCntMin}
	# and $Conf{FullAgeMax}.
        #
	# First make sure that $Conf{FullAgeMax} is at least bigger
	# than $Conf{FullPeriod} * $Conf{FullKeepCnt}, including
	# the exponential array case.
        #
	my $fullKeepCnt = $Conf{FullKeepCnt};
	$fullKeepCnt = [$fullKeepCnt] if ( ref($fullKeepCnt) ne "ARRAY" );
	my $fullAgeMax;
	my $fullPeriod = int(0.5 + $Conf{FullPeriod});
        $fullPeriod = 7 if ( $fullPeriod <= 0 );
	for ( my $i = 0 ; $i < @$fullKeepCnt ; $i++ ) {
	    $fullAgeMax += $fullKeepCnt->[$i] * $fullPeriod;
	    $fullPeriod *= 2;
	}
	$fullAgeMax += $fullPeriod;	# add some buffer

        if ( $cntFull > $Conf{FullKeepCntMin}
               && $oldestFull > $Conf{FullAgeMax}
               && $oldestFull > $fullAgeMax
	       && $Conf{FullKeepCntMin} > 0
	       && $Conf{FullAgeMax} > 0 ) {
            #
            # Only delete a full backup if the Conf settings are satisfied.
            #
            # For pre-V4 we also must make sure that either this backup is the
            # most recent one, or the next backup is filled.
            # (In pre-V4 we can't deleted a full backup if the next backup is not
            # filled.)
            # 
            if ( !$Backups[$firstFull]{preV4} || (@Backups <= $firstFull + 1
                        || !$Backups[$firstFull + 1]{noFill}) ) {
                print("removing filled backup $Backups[$firstFull]{num}\n");
                $XferLOG->write(\"removing filled backup $Backups[$firstFull]{num}\n") if ( $XferLOG );
                last if ( BackupRemove($client, $firstFull, 1) );
                $changes++;
                next;
            }
        }

        #
        # Do new-style full backup expiry, which includes the the case
	# where $Conf{FullKeepCnt} is an array.
        #
        last if ( !BackupFullExpire($client, \@Backups) );
        $changes++;
    }
    $bpc->BackupInfoWrite($client, @Backups) if ( $changes );
}

#
# Handle full backup expiry, using exponential periods.
#
sub BackupFullExpire
{
    my($client, $Backups) = @_;
    my $fullCnt = 0;
    my $fullPeriod = $Conf{FillCycle} <= 0 ? $Conf{FullPeriod} : $Conf{FillCycle};
    my $origFullPeriod = $fullPeriod;
    my $fullKeepCnt = $Conf{FullKeepCnt};
    my $fullKeepIdx = 0;
    my(@delete, @fullList);

    #
    # Don't delete anything if $Conf{FillCycle}, $Conf{FullPeriod} or $Conf{FullKeepCnt}
    # are not defined - possibly a corrupted config.pl file.
    #
    return if ( !defined($Conf{FillCycle}) || !defined($Conf{FullPeriod})
                                           || !defined($Conf{FullKeepCnt}) );

    #
    # If regular backups are still disabled with $Conf{FullPeriod} < 0,
    # we still expire backups based on a typical FullPeriod value - weekly.
    #
    $fullPeriod = 7 if ( $fullPeriod <= 0 );

    $fullKeepCnt = [$fullKeepCnt] if ( ref($fullKeepCnt) ne "ARRAY" );

    for ( my $i = 0 ; $i < @$Backups ; $i++ ) {
        if ( $Backups[$i]{preV4} ) {
            next if ( $Backups->[$i]{type} ne "full" );
        } else {
            next if ( $Backups->[$i]{noFill} );
        }
        push(@fullList, $i);
    }
    for ( my $k = @fullList - 1 ; $k >= 0 ; $k-- ) {
        my $i = $fullList[$k];
        my $prevFull = $fullList[$k-1] if ( $k > 0 );
        #
        # For pre-V4 don't delete any full that is followed by an unfilled backup,
        # since it is needed for restore.
        #
        my $noDelete = $i + 1 < @$Backups ? $Backups->[$i+1]{noFill} : 0;
        $noDelete = 0 if ( !$Backups[$i]{preV4} );

        if ( !$noDelete && 
              ($fullKeepIdx >= @$fullKeepCnt
              || $k > 0
                 && $fullKeepIdx > 0
                 && $Backups->[$i]{startTime} - $Backups->[$prevFull]{startTime}
                             < ($fullPeriod - $origFullPeriod / 2) * 24 * 3600
               )
            ) {
            #
            # Delete the full backup
            #
            #print("Deleting backup $i ($prevFull)\n");
            unshift(@delete, $i);
        } else {
            $fullCnt++;
            while ( $fullKeepIdx < @$fullKeepCnt
                     && $fullCnt >= $fullKeepCnt->[$fullKeepIdx] ) {
                $fullKeepIdx++;
                $fullCnt = 0;
                $fullPeriod = 2 * $fullPeriod;
            }
        }
    }
    #
    # Now actually delete the backups
    #
    for ( my $i = @delete - 1 ; $i >= 0 ; $i-- ) {
        print("removing filled backup $Backups->[$delete[$i]]{num}\n");
        $XferLOG->write(\"removing filled backup $Backups->[$delete[$i]]{num}\n") if ( $XferLOG );
        BackupRemove($client, $delete[$i], 1);
    }
    return @delete;
}

This test script illustrates potential unexpected loss of a few backups after modifying BackupPC settings.

Let incrementals are disabled and we have only full backups.

$Conf{FullKeepCnt}

If we create a new backup and then do expiry check every $Conf{FullPeriod} (or every day if $Conf{FullPeriod} = 1) then on 128 day we will have 18 backups as expected for $Conf{FullKeepCnt} = [ 3, 0, 0, 15 ] :
8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 125, 126, 127

Say we have full backups saved for every day, for instance in case $Conf{FullKeepCnt} = [ 128 ].
After backup #126 we are changing $Conf{FullKeepCnt} to [ 3, 0, 0, 15 ] and doing expire, we will have only 4:
0, 125, 126, 127

Most backups are nuked!

$Conf{FullPeriod}

Let $Conf{FullKeepCnt} = [ 3, 0, 0, 15 ].
If after backup #126 we will change $Conf{FullPeriod} = 1 to $Conf{FullPeriod} = 2 then we will have:
8, 125, 126, 127

$Conf{FullKeepCnt} 2^0 entry

The BackupPC documentation states:

  $Conf{FullKeepCnt} = [4, 2, 3];

Entry #n specifies how many fulls to keep at an interval of 2^n * $Conf{FillCycle} (ie: 1, 2, 4, 8, 16, 32, ...).

The example above specifies keeping 4 of the most recent full backups (1 week interval) two full backups at 2 week intervals, and 3 full backups at 4 week intervals...

Actually, it is true for #n > 0 only. For #n = 0 (2^0) n the most recent full backups will be kept. For instance if 4 manual backups have taken during $Conf{FillCycle}, other backups with 1 * $Conf{FillCycle} interval will be expired.