Skip to content

Instantly share code, notes, and snippets.

@kiyotune
Last active November 2, 2019 17:12
Show Gist options
  • Save kiyotune/350cea9e864eb231b3e7564b16793314 to your computer and use it in GitHub Desktop.
Save kiyotune/350cea9e864eb231b3e7564b16793314 to your computer and use it in GitHub Desktop.
FIND := find -L ./ -name "*.mrc"
MRCS := $(shell $(FIND))
TIFS := $(MRCS:.mrc=.tif)
MRC2TIF := /usr/local/IMOD/bin/mrc2tif -s -c lzw
LIST_MRC:= ./list_mrc.txt
LIST_TIF:= ./list_tif.txt
LISTS := $(LIST_MRC) $(LIST_TIF)
DEL_SRC := 0
define filelist
@ls -al --time-style="+%Y%m%d%H%M%S" $(1) | awk '{print $$5,$$6,$$7}' 1>> $(2)
endef
define calcsize
@cat ./list_$(1).txt | perl -anl -MMath::BigInt -E 'BEGIN{$$sum=0}; $$sum+=Math::BigInt->new([split]->[0]); END{say sprintf("%.2f TB (%d byte)", $$sum/(1024**4), $$sum)}'
endef
.PHONY: all debug calc-size clean-results clean-sources
all: $(TIFS)
@echo mrc2tif is finished !
%.tif: %.mrc
$(MRC2TIF) $< $@
$(call filelist,$<,$(LIST_MRC))
$(call filelist,$@,$(LIST_TIF))
ifeq ($(strip $(DEL_SRC)),1)
rm -f $<
endif
debug:
@echo [command] $(MRC2TIF)
@echo [mrc files]
@echo $(wordlist 1,10, $(MRCS)) ...
@echo [tif files]
@echo $(wordlist 1,10, $(TIFS)) ...
calc-size:
$(call calcsize,'mrc')
$(call calcsize,'tif')
clean-results:
@read -p "clean-results OK? ([Return] to run, [Ctrl+c] to abort)"
@cat $(LIST_TIF) | perl -anl -E 'unlink $$file=[split]->[2]; say "deleted... $$file"'
rm -f $(LISTS)
clean-sources:
@read -p "clean-sources OK? ([Return] to run, [Ctrl+c] to abort)"
@cat $(LIST_MRC) | perl -anl -E 'unlink $$file=[split]->[2]; say "deleted... $$file"'
@kiyotune
Copy link
Author

kiyotune commented Nov 2, 2019

Intel Xeon E5-2690 2.3GHz、RAID10 400TB のファイルサーバー上のディレクトリをSamba共有したものに対して実行したときの例です。処理スピードはディスク書き込みスピードによって大きく変化します。makeコマンドの -j オプションで並列処理数を増やしてもほとんど変化はありませんでした。

  • 変換元 mrc :: 272,761 files, 217 TB
  • 変換後 tif :: 151 TB
  • かかった時間(日数) :: 30 days
  • 圧縮率 :: 69.6 %

@kiyotune
Copy link
Author

kiyotune commented Nov 2, 2019

配布用QRコード
Makefilemrc2tiff

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment