R: data.table::fread's ability to use unix-cmdtool to "stream in" data

command-line-piping.R

# see: https://www.youtube.com/watch?v=RYhwZW6ofbI&t=6s
#      R tip #3: use pipe connections
# by jim Hester

library(data.table)  # for files <  ~10GB
library(skimr)       # another summary()

# filter a file: get only lines containing word UNK


# same, but requires no 3rd Party Code dependencies:
temp3 <- read.csv(pipe( "grep -w UNK bigfile.csv"))

# prefilter 30-column-table, directly from internet
# extremely fast
temp2 <- fread(
              "curl -s https://ftp.ncbi.nlm.nih.gov/genomes/ASSEMBLY_REPORTS/assembly_summary_genbank.txt | head -n 20",
              sep = "\t",
              header = TRUE
              )
skim(temp2)

# same, but requires no 3rd Party Code dependencies:
# slow, downloads entire file first
temp4 <- read.csv(url( "https://ftp.ncbi.nlm.nih.gov/genomes/ASSEMBLY_REPORTS/assembly_summary_genbank.txt"))

temp5 <- read.csv(
    pipe("curl -s https://ftp.ncbi.nlm.nih.gov/genomes/ASSEMBLY_REPORTS/assembly_summary_genbank.txt | 
          perl -MEnglish -ne 'print if $INPUT_LINE_NUMBER == 1 || /Arabidopsis/i'"), sep="\t", header=TRUE
    )


# Help / Documentation
?base::connections
??pipe 



# pipe()