Skip to content

Instantly share code, notes, and snippets.

@coolaj86
Created September 14, 2023 17:50
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save coolaj86/b416e1f18b8ee0179d36d9e7156056b6 to your computer and use it in GitHub Desktop.
Save coolaj86/b416e1f18b8ee0179d36d9e7156056b6 to your computer and use it in GitHub Desktop.

I'm creating a backup retention strategy with this directory structure:

backups
├── daily
├── monthly
│  ├── 2019
│  └── 2020
└── weekly

The monthly directory should keep just one backup per each month of the year, with a subdirectory for the year (2019 and 2020 in the example above).

The weekly directory should keep just one backup per each week of the last 90 days.

The daily directory is where new backups are deposited every 8 hours by our existing backup program.

Write a Go program that will:

  1. read the files in the daily directory and, based on the modified date of the file, do one of the following for each file:
  • leave the file in place if it is less than 72 hours old
  • move it to the weekly folder if a backup for that calendar week does not exist
  • delete it
  1. read the files in the weekly directory and, based on the modified date of the file, do one of the following for each file:
  • leave it in place if it is less than 90 days old, or belongs to the week 90 days ago
  • move it to the monthly/{year}/ directory if a monthly backup for that year does not already exist
  • delete it
package main
import (
"fmt"
"os"
"path/filepath"
"time"
)
const (
dailyDir = "backups/daily"
weeklyDir = "backups/weekly"
monthlyDir = "backups/monthly"
dailyExpire = 72 * time.Hour
weeklyExpire = 90 * 24 * time.Hour
)
func main() {
currentDay := time.Now()
weeklyCutoff := currentDay.Add(-weeklyExpire)
// Process daily backups
err := processDailyBackups(currentDay, weeklyCutoff)
if err != nil {
fmt.Println("Error processing daily backups:", err)
}
// Process weekly backups
err = processWeeklyBackups(currentDay)
if err != nil {
fmt.Println("Error processing weekly backups:", err)
}
}
func processDailyBackups(currentDay, weeklyCutoff time.Time) error {
files, err := ioutilReadDir(dailyDir)
if err != nil {
return err
}
for _, file := range files {
if file.IsDir() {
continue
}
modifiedTime := file.ModTime()
age := currentDay.Sub(modifiedTime)
if age < dailyExpire {
// Leave the file in place if it's less than 72 hours old
continue
}
weekNumber := int(currentDay.YearDay()/7) + 1
weeklyBackupDir := filepath.Join(weeklyDir, fmt.Sprintf("week%d", weekNumber))
if _, err := os.Stat(weeklyBackupDir); os.IsNotExist(err) {
// Move to the weekly folder if a backup for that week does not exist
os.MkdirAll(weeklyBackupDir, os.ModePerm)
err := os.Rename(filepath.Join(dailyDir, file.Name()), filepath.Join(weeklyBackupDir, file.Name()))
if err != nil {
fmt.Printf("Error moving %s to weekly: %v\n", file.Name(), err)
}
} else {
// Delete the file if it's older than 72 hours and a weekly backup already exists
err := os.Remove(filepath.Join(dailyDir, file.Name()))
if err != nil {
fmt.Printf("Error deleting %s: %v\n", file.Name(), err)
}
}
}
return nil
}
func processWeeklyBackups(currentDay time.Time) error {
files, err := ioutilReadDir(weeklyDir)
if err != nil {
return err
}
for _, file := range files {
if file.IsDir() {
continue
}
modifiedTime := file.ModTime()
age := currentDay.Sub(modifiedTime)
if age < weeklyExpire {
// Leave it in place if it's less than 90 days old or belongs to the week 90 days ago
continue
}
year := modifiedTime.Year()
monthlyYearDir := filepath.Join(monthlyDir, fmt.Sprintf("%d", year))
os.MkdirAll(monthlyYearDir, os.ModePerm)
monthlyBackupFile := filepath.Join(monthlyYearDir, file.Name())
if _, err := os.Stat(monthlyBackupFile); os.IsNotExist(err) {
// Move to the monthly/{year}/ directory if a monthly backup for that year does not already exist
err := os.Rename(filepath.Join(weeklyDir, file.Name()), monthlyBackupFile)
if err != nil {
fmt.Printf("Error moving %s to monthly: %v\n", file.Name(), err)
}
} else {
// Delete the file if it's older than 90 days and a monthly backup already exists
err := os.Remove(filepath.Join(weeklyDir, file.Name()))
if err != nil {
fmt.Printf("Error deleting %s: %v\n", file.Name(), err)
}
}
}
return nil
}
func ioutilReadDir(dirPath string) ([]os.FileInfo, error) {
files, err := os.ReadDir(dirPath)
if err != nil {
return nil, err
}
var result []os.FileInfo
for _, file := range files {
result = append(result, file)
}
return result, nil
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment