Decompression speeds with some different libraries / interfaces

decompression_speeds.md

$ uname -a
Linux fern 6.2.0-33-generic #33~22.04.1-Ubuntu SMP PREEMPT_DYNAMIC Thu Sep  7 10:33:52 UTC 2 x86_64 x86_64 x86_64 GNU/Linux

zcat

$ zcat data/20201028_CCDG_14151_B01_GRM_WGS_2020-08-05_chr21.recalibrated_variants.vcf.gz|head -c $(( 10 * 1024 * 1024 * 1024 )) | pv >/dev/null
10.0GiB 0:00:47 [ 216MiB/s] [                                                                                                                                                           <=>       ]

rust program (with standard-zlib)

$ ./target/release/fast_unzip data/20201028_CCDG_14151_B01_GRM_WGS_2020-08-05_chr21.recalibrated_variants.vcf.gz | pv > /dev/null
10.0GiB 0:00:20 [ 489MiB/s] [                                                                    <=>                                                                                              ]

rust program (with cloudflare-zlib)

$ ./target/release/fast_unzip data/20201028_CCDG_14151_B01_GRM_WGS_2020-08-05_chr21.recalibrated_variants.vcf.gz | pv > /dev/null
10.0GiB 0:00:12 [ 795MiB/s] [                                          <=>                                                                                                                        ]

RapidGzip (16 threads)

$ ./src/tools/rapidgzip -d -c -P 16 ../../fast_unzip/data/20201028_CCDG_14151_B01_GRM_WGS_2020-08-05_chr21.recalibrated_variants.vcf.gz | head -c $(( 10 * 1024 * 1024 * 1024 )) | pv >/dev/null
10.0GiB 0:00:09 [1.09GiB/s] [                                <=>                                                                                                                                  ]