Go code to concurrently recursively walk file tree counting files, spawning new Goroutine per sub directory found.

concurrent-recursive-file-size-counter.go

// Entry point call, pass in valid directory path and nil file pointer

total, err := getDirSize("/root/dir/to/count/from", nil)

// will either get empty string and pointer or filled string with nil pointer
// depending on whether it needs to just count the files still remaining in
// this given dir or whether it needs to start counting again in a found sub dir
func getDirSize(path string, filePtr *os.File) (int64, error) {
	var total int64

	fp, err := resolveFilePointer(path, filePtr)
	if err != nil {
		return total, err
	}

	files, err := fp.Readdir(100)
	if len(files) == 0 {
		return total, err
	}

	total += countFileSizes(files, func(f os.FileInfo) int64 {
		done := make(chan interface{})

		var total int64
		go func(d chan interface{}, t *int64) {
			s, err := getDirSize(fmt.Sprintf("%s%c%s", path, os.PathSeparator, f.Name()), nil)
			if err != nil {
				logging.Error("Unable to get dirs full size: %v...", err)
			}
			*t += s
			close(done)
		}(done, &total)

		<-done

		return total
	})

	if err != io.EOF {
		t, err := getDirSize("", fp)
		if err == nil {
			total += t
		}
	}

	return total, nil
}

func countFileSizes(files []os.FileInfo, onDirFile func(os.FileInfo) int64) int64 {
	var total int64
	for _, f := range files {
		if f.IsDir() {
			total += onDirFile(f)
			continue
		}
		if f.Mode().IsRegular() {
			total += f.Size()
		}
	}
	return total
}

func resolveFilePointer(path string, file *os.File) (*os.File, error) {
	var fp *os.File
	fp = file
	if fp == nil {
		filePtr, err := os.Open(path)
		if err != nil {
			return nil, err
		}
		fp = filePtr
	}
	return fp, nil
}