Check tar files status

.gitignore

/tarcheck
/elsevier_tarcheck.ndj

README.md

tarcheck

Mass tarball file checks.

Takes one tar filename per line from stdin and reports any errors found in a JSON lines file.

We had to check 1588 tar file containing millions of files. This program takes about 3 minutes and is for some reason much faster than tar tf plus exit code checks, plus the report here is JSON.

$ echo "myfile.tar" | tarcheck

Makefile

SHELL := /bin/bash

elsevier_tarcheck.ndj: tarcheck
	# takes about 3 minutes
	taskcat ElsevierJournalsPaths | egrep 'tar$$' | ./tarcheck > $@

tarcheck: tarcheck.go
	go build -o $@ $^

.PHONY: clean
clean:
	rm tarcheck

tarcheck.go

package main

import (
	"archive/tar"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"os"
	"strings"

	"github.com/miku/parallel"
)

type Result struct {
	Filename string   `json:"f"`
	Errs     []string `json:"errs"`
	Contents []string `json:"c"`
}

func main() {
	pp := parallel.NewProcessor(os.Stdin, os.Stdout, func(p []byte) ([]byte, error) {
		filename := strings.TrimSpace(string(p))
		log.Println(filename)
		f, err := os.Open(filename)
		if err != nil {
			return nil, err
		}
		defer f.Close()
		result := Result{
			Filename: filename,
			Errs:     make([]string, 0),
			Contents: make([]string, 0),
		}
		tr := tar.NewReader(f)
		for {
			hdr, err := tr.Next()
			if err == io.EOF {
				break // End of archive
			}
			if err != nil {
				result.Errs = append(result.Errs, fmt.Sprintf("%v", err))
				break
			}
			if hdr == nil {
				result.Errs = append(result.Errs, "unreadable tar")
				break
			}
			result.Contents = append(result.Contents, hdr.Name)
		}
		b, err := json.Marshal(result)
		b = append(b, []byte("\n")...)
		return b, err
	})
	if err := pp.Run(); err != nil {
		log.Fatal(err)
	}
}