parallel_crc32.go

package main

import (
	"hash/crc32"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"sync"
)

type result struct {
	path string
	sum string
	err error
}

func sumFiles(crc32table *crc32.Table, root string) (<-chan result, <-chan error) {
	results := make(chan result, 32)
	error_channel := make(chan error, 1)

	go func(results chan result, error_channel chan error) {
		var wg sync.WaitGroup

		err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
			if err != nil {
				results <- result{path, "ERR", nil}
				return nil
			}

			if info.IsDir() {
				results <- result{path, "DIR", nil}
				return nil
			}

			if (info.Mode() & os.ModeSymlink == os.ModeSymlink) {
				results <- result{path, "SYMLINK", nil}
				return nil
			}
			
			results <- result{path, "START_FILE", nil}

			wg.Add(1)

			go func(path string, c chan result, wg *sync.WaitGroup) { // HL
				f, err := os.Open(path)

				data := []byte{}

				if err == nil {
					data, err = ioutil.ReadAll(f)	
				}

				f.Close()
				
				results <- result{path, fmt.Sprintf("%10d", crc32.Checksum(data, crc32table)), err}

				wg.Done()
			}(path, results, &wg)

			return nil
		})

		go func(<-chan result) {
			wg.Wait()
			close(results)
		}(results)
		// No select needed here, since errc is buffered.
		error_channel <- err
	}(results, error_channel)

	return results, error_channel
}

func main() {
	crc32table := crc32.MakeTable(crc32.Koopman)

	fsRoot := os.Args[1]
	fmt.Printf("walking path\t%s\n", fsRoot)
	
	results, _ := sumFiles(crc32table, fsRoot)

	for single_file_result := range results {
		if single_file_result.err != nil {
			fmt.Printf("ERR %s\t%s\t%s\n", single_file_result.sum, single_file_result.path, single_file_result.err)
		} else {
			fmt.Printf("%10s\t%s\n", single_file_result.sum, single_file_result.path)
		}
		
	}
}