Skip to content

Instantly share code, notes, and snippets.

@phonedude

phonedude/content-length-last-modified-ia-gcp-aws.txt

Created July 9, 2021 15:39
Show Gist options
  • Save phonedude/0ca25f75ee67f37d83e505d247424dbd to your computer and use it in GitHub Desktop.
Save phonedude/0ca25f75ee67f37d83e505d247424dbd to your computer and use it in GitHub Desktop.
Download ZIP
Content-Length and Last-Modified of IA, GCP, and AWS files from: https://www.biorxiv.org/content/10.1101/2021.06.18.449051v1
Raw
content-length-last-modified-ia-gcp-aws.txt
sh-3.2$ more uri-aws.txt
#!/bin/csh -x
curl -iLs https://sra-pub-sars-cov2.s3.amazonaws.com/run/SRR11313485/SRR11313485 > SRR11313485.aws.txt
curl -iLs https://sra-pub-sars-cov2.s3.amazonaws.com/run/SRR11313486/SRR11313486 > SRR11313486.aws.txt
curl -iLs https://sra-pub-sars-cov2.s3.amazonaws.com/run/SRR11313274/SRR11313274 > SRR11313274.aws.txt
curl -iLs https://sra-pub-sars-cov2.s3.amazonaws.com/run/SRR11313275/SRR11313275 > SRR11313275.aws.txt
curl -iLs https://sra-pub-sars-cov2.s3.amazonaws.com/run/SRR11313285/SRR11313285 > SRR11313285.aws.txt
curl -iLs https://sra-pub-sars-cov2.s3.amazonaws.com/run/SRR11313286/SRR11313286 > SRR11313286.aws.txt
curl -iLs https://sra-pub-sars-cov2.s3.amazonaws.com/run/SRR11313448/SRR11313448 > SRR11313448.aws.txt
curl -iLs https://sra-pub-sars-cov2.s3.amazonaws.com/run/SRR11313449/SRR11313449 > SRR11313449.aws.txt
curl -iLs https://sra-pub-sars-cov2.s3.amazonaws.com/run/SRR11313427/SRR11313427 > SRR11313427.aws.txt
curl -iLs https://sra-pub-sars-cov2.s3.amazonaws.com/run/SRR11313429/SRR11313429 > SRR11313429.aws.txt
sh-3.2$ more uri-gcp.txt
#!/bin/csh -x
curl -iLs https://storage.googleapis.com/nih-sequence-read-archive/run/SRR11313485/SRR11313485 > SRR11313485.gcp.txt
curl -iLs https://storage.googleapis.com/nih-sequence-read-archive/run/SRR11313486/SRR11313486 > SRR11313486.gcp.txt
curl -iLs https://storage.googleapis.com/nih-sequence-read-archive/run/SRR11313274/SRR11313274 > SRR11313274.gcp.txt
curl -iLs https://storage.googleapis.com/nih-sequence-read-archive/run/SRR11313275/SRR11313275 > SRR11313275.gcp.txt
curl -iLs https://storage.googleapis.com/nih-sequence-read-archive/run/SRR11313285/SRR11313285 > SRR11313285.gcp.txt
curl -iLs https://storage.googleapis.com/nih-sequence-read-archive/run/SRR11313286/SRR11313286 > SRR11313286.gcp.txt
curl -iLs https://storage.googleapis.com/nih-sequence-read-archive/run/SRR11313448/SRR11313448 > SRR11313448.gcp.txt
curl -iLs https://storage.googleapis.com/nih-sequence-read-archive/run/SRR11313449/SRR11313449 > SRR11313449.gcp.txt
curl -iLs https://storage.googleapis.com/nih-sequence-read-archive/run/SRR11313427/SRR11313427 > SRR11313427.gcp.txt
curl -iLs https://storage.googleapis.com/nih-sequence-read-archive/run/SRR11313429/SRR11313429 > SRR11313429.gcp.txt
sh-3.2$ more uri-ia-raw.txt
#!/bin/csh -x
curl -iLs http://web.archive.org/web/1id_/https://storage.googleapis.com/nih-sequence-read-archive/run/SRR11313485/SRR11313485 > SRR11313485.ia.txt
curl -iLs http://web.archive.org/web/1id_/https://storage.googleapis.com/nih-sequence-read-archive/run/SRR11313486/SRR11313486 > SRR11313486.ia.txt
curl -iLs http://web.archive.org/web/1id_/https://storage.googleapis.com/nih-sequence-read-archive/run/SRR11313274/SRR11313274 > SRR11313274.ia.txt
curl -iLs http://web.archive.org/web/1id_/https://storage.googleapis.com/nih-sequence-read-archive/run/SRR11313275/SRR11313275 > SRR11313275.ia.txt
curl -iLs http://web.archive.org/web/1id_/https://storage.googleapis.com/nih-sequence-read-archive/run/SRR11313285/SRR11313285 > SRR11313285.ia.txt
curl -iLs http://web.archive.org/web/1id_/https://storage.googleapis.com/nih-sequence-read-archive/run/SRR11313286/SRR11313286 > SRR11313286.ia.txt
curl -iLs http://web.archive.org/web/1id_/https://storage.googleapis.com/nih-sequence-read-archive/run/SRR11313448/SRR11313448 > SRR11313448.ia.txt
curl -iLs http://web.archive.org/web/1id_/https://storage.googleapis.com/nih-sequence-read-archive/run/SRR11313449/SRR11313449 > SRR11313449.ia.txt
curl -iLs http://web.archive.org/web/1id_/https://storage.googleapis.com/nih-sequence-read-archive/run/SRR11313427/SRR11313427 > SRR11313427.ia.txt
curl -iLs http://web.archive.org/web/1id_/https://storage.googleapis.com/nih-sequence-read-archive/run/SRR11313429/SRR11313429 > SRR11313429.ia.txt
sh-3.2$ grep --binary-files=text "Last-Modified:" SRR*txt
SRR11313274.aws.txt:Last-Modified: Thu, 18 Jun 2020 19:44:51 GMT
SRR11313274.gcp.txt:Last-Modified: Tue, 02 Feb 2021 18:54:48 GMT
SRR11313274.ia.txt:X-Archive-Orig-Last-Modified: Tue, 02 Feb 2021 18:54:48 GMT
SRR11313275.aws.txt:Last-Modified: Thu, 18 Jun 2020 20:07:41 GMT
SRR11313275.gcp.txt:Last-Modified: Tue, 02 Feb 2021 18:57:30 GMT
SRR11313275.ia.txt:X-Archive-Orig-Last-Modified: Tue, 02 Feb 2021 18:57:30 GMT
SRR11313285.aws.txt:Last-Modified: Sun, 14 Jun 2020 05:06:57 GMT
SRR11313285.gcp.txt:Last-Modified: Tue, 02 Feb 2021 18:54:51 GMT
SRR11313285.ia.txt:X-Archive-Orig-Last-Modified: Tue, 02 Feb 2021 18:54:51 GMT
SRR11313286.aws.txt:Last-Modified: Thu, 18 Jun 2020 19:42:17 GMT
SRR11313286.gcp.txt:Last-Modified: Tue, 02 Feb 2021 18:58:55 GMT
SRR11313286.ia.txt:X-Archive-Orig-Last-Modified: Tue, 02 Feb 2021 18:58:55 GMT
SRR11313427.aws.txt:Last-Modified: Thu, 18 Jun 2020 20:20:06 GMT
SRR11313427.gcp.txt:Last-Modified: Tue, 02 Feb 2021 18:48:05 GMT
SRR11313427.ia.txt:X-Archive-Orig-Last-Modified: Tue, 02 Feb 2021 18:48:05 GMT
SRR11313429.aws.txt:Last-Modified: Thu, 18 Jun 2020 20:51:11 GMT
SRR11313429.gcp.txt:Last-Modified: Tue, 02 Feb 2021 18:55:50 GMT
SRR11313429.ia.txt:X-Archive-Orig-Last-Modified: Tue, 02 Feb 2021 18:55:50 GMT
SRR11313448.aws.txt:Last-Modified: Thu, 18 Jun 2020 19:27:37 GMT
SRR11313448.gcp.txt:Last-Modified: Tue, 02 Feb 2021 18:47:48 GMT
SRR11313448.ia.txt:X-Archive-Orig-Last-Modified: Tue, 02 Feb 2021 18:47:48 GMT
SRR11313449.aws.txt:Last-Modified: Thu, 18 Jun 2020 21:29:03 GMT
SRR11313449.gcp.txt:Last-Modified: Tue, 02 Feb 2021 18:59:04 GMT
SRR11313449.ia.txt:X-Archive-Orig-Last-Modified: Tue, 02 Feb 2021 18:59:04 GMT
SRR11313485.aws.txt:Last-Modified: Thu, 18 Jun 2020 19:14:16 GMT
SRR11313485.gcp.txt:Last-Modified: Tue, 02 Feb 2021 18:47:49 GMT
SRR11313485.ia.txt:X-Archive-Orig-Last-Modified: Tue, 02 Feb 2021 18:47:49 GMT
SRR11313486.aws.txt:Last-Modified: Thu, 18 Jun 2020 21:03:39 GMT
SRR11313486.gcp.txt:Last-Modified: Tue, 02 Feb 2021 18:58:04 GMT
SRR11313486.ia.txt:X-Archive-Orig-Last-Modified: Tue, 02 Feb 2021 18:58:04 GMT
sh-3.2$ grep --binary-files=text "Content-Length:" SRR*txt | grep -v "ia.txt:Content-Length:"
SRR11313274.aws.txt:Content-Length: 1425203
SRR11313274.gcp.txt:Content-Length: 1425203
SRR11313274.ia.txt:X-Archive-Orig-Content-Length: 1425203
SRR11313275.aws.txt:Content-Length: 209675
SRR11313275.gcp.txt:Content-Length: 209675
SRR11313275.ia.txt:X-Archive-Orig-Content-Length: 209675
SRR11313285.aws.txt:Content-Length: 13607640
SRR11313285.gcp.txt:Content-Length: 13607640
SRR11313285.ia.txt:X-Archive-Orig-Content-Length: 13607640
SRR11313286.aws.txt:Content-Length: 1113480
SRR11313286.gcp.txt:Content-Length: 1113480
SRR11313286.ia.txt:X-Archive-Orig-Content-Length: 1113480
SRR11313427.aws.txt:Content-Length: 404143
SRR11313427.gcp.txt:Content-Length: 404143
SRR11313427.ia.txt:X-Archive-Orig-Content-Length: 404143
SRR11313429.aws.txt:Content-Length: 104487
SRR11313429.gcp.txt:Content-Length: 104487
SRR11313429.ia.txt:X-Archive-Orig-Content-Length: 104487
SRR11313448.aws.txt:Content-Length: 661901
SRR11313448.gcp.txt:Content-Length: 661901
SRR11313448.ia.txt:X-Archive-Orig-Content-Length: 661901
SRR11313449.aws.txt:Content-Length: 113935
SRR11313449.gcp.txt:Content-Length: 113935
SRR11313449.ia.txt:X-Archive-Orig-Content-Length: 113935
SRR11313485.aws.txt:Content-Length: 446341
SRR11313485.gcp.txt:Content-Length: 446341
SRR11313485.ia.txt:X-Archive-Orig-Content-Length: 446341
SRR11313486.aws.txt:Content-Length: 102067
SRR11313486.gcp.txt:Content-Length: 102067
SRR11313486.ia.txt:X-Archive-Orig-Content-Length: 102067
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment