目录文件去重

Duplicate_file_removal.go

package main

import (
	"crypto/md5"
	"encoding/hex"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
)

var Md5List = make(map[string]int)
var count = 0
var errorCount = 0

func pp(path string, f os.FileInfo, err error) error {
	if f.Mode().IsRegular() {
		md5Value := fileMD5(path)
		if _, ok := Md5List[md5Value]; ok {
			fmt.Printf("Remove File:%s \n", path)
			err := os.Remove(path)
			if err != nil {
				log.Fatal(err)
				errorCount++
			} else {
				count++
			}

		} else {
			Md5List[md5Value] = 1
		}
	}
	return nil
}

func fileMD5(path string) string {
	f, err := os.Open(path)
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	h := md5.New()
	if _, err := io.Copy(h, f); err != nil {
		log.Fatal(err)
	}
	a := h.Sum(nil)
	return hex.EncodeToString(a[:])
}

func main() {
	Md5List = make(map[string]int)
	aKey := flag.String("path", "", "去重目录")
	flag.Parse()
	filePath := *aKey
	err := filepath.Walk(filePath, pp)
	fmt.Printf("去重完成！重复文件：%v 错误处理文件： %v Walk错误： %v\n", count, errorCount, err)
}