Makefile for chunking the Los Alamos dataset for parallel processing of the data, while preserving compression.

Makefile

SHELL = /bin/bash
bigfiles = auth dns proc flows
lines-per-chunk = 250000

.PHONY: bigfiles
bigfiles: $(addsuffix .chunks,$(bigfiles))

%.chunks: %.txt.gz
	trap "rm -rf $@" INT ERR;\
		mkdir $@;\
		zcat $< |\
		    split --verbose\
                  -d\
                  --suffix-length=4\
                  --lines=$(lines-per-chunk)\
                  --additional-suffix=.txt.gz\
                  --filter='gzip -c >$$FILE'\
                  -\
                  '$@/'

.PHONY: clean
clean:
	rm -rf *.chunks