duplicacy_snapshotmanager.go

type chunkStats struct {
	// Stores the chunk file size for each chunk
	size int64

	// Indicate whether or not a chunk is shared by multiple snapshots
	unique bool

	// Store the index of the snapshot that references each chunk; if the chunk is shared by multiple chunks, the index is -1
	snapshotIndex int
}

func (manager *SnapshotManager) CheckSnapshots(snapshotID string, revisionsToCheck []int, tag string, showStatistics bool, showTabular bool,
	checkFiles bool, searchFossils bool, resurrect bool) bool {

	LOG_DEBUG("LIST_PARAMETERS", "id: %s, revisions: %v, tag: %s, showStatistics: %t, showTabular: %t, checkFiles: %t, searchFossils: %t, resurrect: %t",
		snapshotID, revisionsToCheck, tag, showStatistics, showTabular, checkFiles, searchFossils, resurrect)

	snapshotMap := make(map[string][]*Snapshot)
	var err error

	// Keep all stats in a single map.
	chunkStatsMap = make(map[string]chunkStats)

	// Stores the chunk file size for each chunk
	chunkSizeMap := make(map[string]int64)

	// Indicate whether or not a chunk is shared by multiple snapshots
	chunkUniqueMap := make(map[string]bool)

	// Store the index of the snapshot that references each chunk; if the chunk is shared by multiple chunks, the index is -1
	chunkSnapshotMap := make(map[string]int)

	LOG_INFO("SNAPSHOT_CHECK", "Listing all chunks")
	allChunks, allSizes := manager.ListAllFiles(manager.storage, chunkDir)

	for i, chunk := range allChunks {
		if len(chunk) == 0 || chunk[len(chunk)-1] == '/' {
			continue
		}

		if strings.HasSuffix(chunk, ".fsl") {
			continue
		}

		chunk = strings.Replace(chunk, "/", "", -1)
		chunkStatsMap[chunk].size = allSizes[i]
	}

	if snapshotID == "" || showStatistics || showTabular {
		snapshotIDs, err := manager.ListSnapshotIDs()
		if err != nil {
			LOG_ERROR("SNAPSHOT_LIST", "Failed to list all snapshots: %v", err)
			return false
		}

		for _, snapshotID := range snapshotIDs {
			snapshotMap[snapshotID] = nil
		}

	} else {
		snapshotMap[snapshotID] = nil
	}

	snapshotIDIndex := 0
	totalMissingChunks := 0
	for snapshotID = range snapshotMap {

		revisions := revisionsToCheck
		if len(revisions) == 0 || showStatistics || showTabular {
			revisions, err = manager.ListSnapshotRevisions(snapshotID)
			if err != nil {
				LOG_ERROR("SNAPSHOT_LIST", "Failed to list all revisions for snapshot %s: %v", snapshotID, err)
				return false
			}
		}

		for _, revision := range revisions {
			snapshot := manager.DownloadSnapshot(snapshotID, revision)
			if tag != "" && snapshot.Tag != tag {
				continue
			}
			snapshotMap[snapshotID] = append(snapshotMap[snapshotID], snapshot)
		}
	}

	totalRevisions := 0
	for _, snapshotList := range snapshotMap {
		totalRevisions += len(snapshotList)
	}
	LOG_INFO("SNAPSHOT_CHECK", "%d snapshots and %d revisions", len(snapshotMap), totalRevisions)

	var totalChunkSize int64
	for _, curChunk := range chunkStatsMap {
		totalChunkSize += curChunk.size
	}
	LOG_INFO("SNAPSHOT_CHECK", "Total chunk size is %s in %d chunks", PrettyNumber(totalChunkSize), len(chunkStatsMap))

	for snapshotID = range snapshotMap {

		for _, snapshot := range snapshotMap[snapshotID] {

			if checkFiles {
				manager.DownloadSnapshotContents(snapshot, nil, false)
				manager.VerifySnapshot(snapshot)
				continue
			}

			chunks := make(map[string]bool)
			for _, chunkID := range manager.GetSnapshotChunks(snapshot, false) {
				chunks[chunkID] = true
			}

			missingChunks := 0
			for chunkID := range chunks {

				_, found := chunkStatsMap[chunkID]

				if !found {

					// Look up the chunk again in case it actually exists, but only if there aren't
					// too many missing chunks.
					if missingChunks < 100 {
						_, exist, _, err := manager.storage.FindChunk(0, chunkID, false)
						if err != nil {
							LOG_WARN("SNAPSHOT_VALIDATE", "Failed to check the existence of chunk %s: %v",
							         chunkID, err)
						} else if exist {
							LOG_INFO("SNAPSHOT_VALIDATE", "Chunk %s is confirmed to exist", chunkID)
							continue
						}
					}

					if !searchFossils {
						missingChunks += 1
						LOG_WARN("SNAPSHOT_VALIDATE",
							"Chunk %s referenced by snapshot %s at revision %d does not exist",
							chunkID, snapshotID, snapshot.Revision)
						continue
					}

					chunkPath, exist, size, err := manager.storage.FindChunk(0, chunkID, true)
					if err != nil {
						LOG_ERROR("SNAPSHOT_VALIDATE", "Failed to check the existence of fossil %s: %v",
							chunkID, err)
						return false
					}

					if !exist {
						missingChunks += 1
						LOG_WARN("SNAPSHOT_VALIDATE",
							"Chunk %s referenced by snapshot %s at revision %d does not exist",
							chunkID, snapshotID, snapshot.Revision)
						continue
					}

					if resurrect {
						manager.resurrectChunk(chunkPath, chunkID)
					} else {
						LOG_WARN("SNAPSHOT_FOSSIL", "Chunk %s referenced by snapshot %s at revision %d "+
							"has been marked as a fossil", chunkID, snapshotID, snapshot.Revision)
					}

					chunkStatsMap[chunkID].size = size
				}

				if unique, found := chunkStatsMap[chunkID].unique; !found {
					chunkStatsMap[chunkID].unique = true
				} else {
					if unique {
						chunkStatsMap[chunkID].unique = false
					}
				}

				if previousSnapshotIDIndex, found := chunkStatsMap[chunkID].snapshotID; !found {
					chunkStatsMap[chunkID].snapshotID = snapshotIDIndex
				} else if previousSnapshotIDIndex != snapshotIDIndex && previousSnapshotIDIndex != -1 {
					chunkStatsMap[chunkID].snapshotID = -1
				}
			}

			if missingChunks > 0 {
				LOG_WARN("SNAPSHOT_CHECK", "Some chunks referenced by snapshot %s at revision %d are missing",
					snapshotID, snapshot.Revision)
				totalMissingChunks += missingChunks
			} else {
				LOG_INFO("SNAPSHOT_CHECK", "All chunks referenced by snapshot %s at revision %d exist",
					snapshotID, snapshot.Revision)
			}
		}

		snapshotIDIndex += 1
	}

	if totalMissingChunks > 0 {
		LOG_ERROR("SNAPSHOT_CHECK", "Some chunks referenced by some snapshots do not exist in the storage")
		return false
	}

	if showTabular {
		manager.ShowStatisticsTabular(snapshotMap, chunkStatsMap)
	} else if showStatistics {
		manager.ShowStatistics(snapshotMap, chunkStatsMap)
	}

	return true
}



func (manager *SnapshotManager) ShowStatistics(snapshotMap map[string][]*Snapshot, chunkStatsMap map[string]chunkStats) {
	for snapshotID, snapshotList := range snapshotMap {

		snapshotChunks := make(map[string]bool)

		for _, snapshot := range snapshotList {

			chunks := make(map[string]bool)
			for _, chunkID := range manager.GetSnapshotChunks(snapshot, false) {
				chunks[chunkID] = true
				snapshotChunks[chunkID] = true
			}

			var totalChunkSize int64
			var uniqueChunkSize int64

			for chunkID := range chunks {
				chunkSize := chunkStatsMap[chunkID].size
				totalChunkSize += chunkSize
				if chunkStatsMap[chunkID].unique {
					uniqueChunkSize += chunkSize
				}
			}

			files := ""
			if snapshot.FileSize != 0 && snapshot.NumberOfFiles != 0 {
				files = fmt.Sprintf("%d files (%s bytes), ", snapshot.NumberOfFiles, PrettyNumber(snapshot.FileSize))
			}
			LOG_INFO("SNAPSHOT_CHECK", "Snapshot %s at revision %d: %s%s total chunk bytes, %s unique chunk bytes",
				snapshot.ID, snapshot.Revision, files, PrettyNumber(totalChunkSize), PrettyNumber(uniqueChunkSize))
		}

		var totalChunkSize int64
		var uniqueChunkSize int64
		for chunkID := range snapshotChunks {
			chunkSize := chunkStatsMap[chunkID].size
			totalChunkSize += chunkSize

			if chunkStatsMap[chunkID].snapshotID != -1 {
				uniqueChunkSize += chunkSize
			}
		}
		LOG_INFO("SNAPSHOT_CHECK", "Snapshot %s all revisions: %s total chunk bytes, %s unique chunk bytes",
			snapshotID, PrettyNumber(totalChunkSize), PrettyNumber(uniqueChunkSize))
	}
}


// Print snapshot and revision statistics in tabular format
func (manager *SnapshotManager) ShowStatisticsTabular(snapshotMap map[string][]*Snapshot, chunkStatsMap map[string]chunkStats) {
	tableBuffer := new(bytes.Buffer)
	tableWriter := tabwriter.NewWriter(tableBuffer, 0, 0, 1, ' ', tabwriter.AlignRight|tabwriter.Debug)

	for snapshotID, snapshotList := range snapshotMap {
		fmt.Fprintln(tableWriter, "")
		fmt.Fprintln(tableWriter, " snap \trev \t \tfiles \tbytes \tchunks \tbytes \tuniq \tbytes \tnew \tbytes \t")
		snapshotChunks := make(map[string]bool)

		earliestSeenChunks := make(map[string]int)

		for _, snapshot := range snapshotList {
			for _, chunkID := range manager.GetSnapshotChunks(snapshot, false) {
				if earliestSeenChunks[chunkID] == 0 {
					earliestSeenChunks[chunkID] = math.MaxInt32
				}
				if earliestSeenChunks[chunkID] > snapshot.Revision {
					earliestSeenChunks[chunkID] = snapshot.Revision
				}
			}
		}

		for _, snapshot := range snapshotList {

			chunks := make(map[string]bool)
			for _, chunkID := range manager.GetSnapshotChunks(snapshot, false) {
				chunks[chunkID] = true
				snapshotChunks[chunkID] = true
			}

			var totalChunkSize int64
			var uniqueChunkSize int64
			var totalChunkCount int64
			var uniqueChunkCount int64
			var newChunkCount int64
			var newChunkSize int64

			for chunkID := range chunks {
				chunkSize := chunkStatsMap[chunkID].size
				totalChunkSize += chunkSize
				totalChunkCount += 1
				if earliestSeenChunks[chunkID] == snapshot.Revision {
					newChunkCount += 1
					newChunkSize += chunkSize
				}
				if chunkStatsMap[chunkID].unique {
					uniqueChunkSize += chunkSize
					uniqueChunkCount += 1
				}
			}

			files := " \t "
			if snapshot.FileSize != 0 && snapshot.NumberOfFiles != 0 {
				files = fmt.Sprintf("%d \t%s", snapshot.NumberOfFiles, PrettyNumber(snapshot.FileSize))
			}
			creationTime := time.Unix(snapshot.StartTime, 0).Format("2006-01-02 15:04")
			fmt.Fprintln(tableWriter, fmt.Sprintf(
				"%s \t%d \t@ %s %5s \t%s \t%d \t%s \t%d \t%s \t%d \t%s \t",
				snapshotID, snapshot.Revision, creationTime, snapshot.Options, files, totalChunkCount, PrettyNumber(totalChunkSize), uniqueChunkCount, PrettyNumber(uniqueChunkSize), newChunkCount, PrettyNumber(newChunkSize)))
		}

		var totalChunkSize int64
		var uniqueChunkSize int64
		var totalChunkCount int64
		var uniqueChunkCount int64
		for chunkID := range snapshotChunks {
			chunkSize := chunkStatsMap[chunkID].size
			totalChunkSize += chunkSize
			totalChunkCount += 1

			if chunkStatsMap[chunkID].snapshotID != -1 {
				uniqueChunkSize += chunkSize
				uniqueChunkCount += 1
			}
		}
		fmt.Fprintln(tableWriter, fmt.Sprintf(
			"%s \tall \t \t \t \t%d \t%s \t%d \t%s \t \t \t",
			snapshotID, totalChunkCount, PrettyNumber(totalChunkSize), uniqueChunkCount, PrettyNumber(uniqueChunkSize)))
	}
	tableWriter.Flush()
	LOG_INFO("SNAPSHOT_CHECK", tableBuffer.String())
}