Skip to content

Instantly share code, notes, and snippets.

@xjoker
Last active May 10, 2017 02:51
Show Gist options
  • Save xjoker/d2a475662b36378d270fc4c22ac98123 to your computer and use it in GitHub Desktop.
Save xjoker/d2a475662b36378d270fc4c22ac98123 to your computer and use it in GitHub Desktop.
目录文件去重
package main
import (
"crypto/md5"
"encoding/hex"
"flag"
"fmt"
"io"
"log"
"os"
"path/filepath"
)
var Md5List = make(map[string]int)
var count = 0
var errorCount = 0
func pp(path string, f os.FileInfo, err error) error {
if f.Mode().IsRegular() {
md5Value := fileMD5(path)
if _, ok := Md5List[md5Value]; ok {
fmt.Printf("Remove File:%s \n", path)
err := os.Remove(path)
if err != nil {
log.Fatal(err)
errorCount++
} else {
count++
}
} else {
Md5List[md5Value] = 1
}
}
return nil
}
func fileMD5(path string) string {
f, err := os.Open(path)
if err != nil {
log.Fatal(err)
}
defer f.Close()
h := md5.New()
if _, err := io.Copy(h, f); err != nil {
log.Fatal(err)
}
a := h.Sum(nil)
return hex.EncodeToString(a[:])
}
func main() {
Md5List = make(map[string]int)
aKey := flag.String("path", "", "去重目录")
flag.Parse()
filePath := *aKey
err := filepath.Walk(filePath, pp)
fmt.Printf("去重完成!重复文件:%v 错误处理文件: %v Walk错误: %v\n", count, errorCount, err)
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment