kevinushey/.gitignore

## .gitignore
.Rproj.user
.Rhistory
.RData
*.Rproj
*.html

## read-file.cpp
#include <fstream>
#include <sstream>
#include <string>
#include <sys/mman.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <Rcpp.h>
using namespace Rcpp;

// [[Rcpp::export]]
CharacterVector read_file_cpp1(std::string path) {
  std::ifstream t(path.c_str());
  std::stringstream ss;
  ss << t.rdbuf();
  return ss.str();
}

// [[Rcpp::export]]
CharacterVector read_file_cpp2(std::string path) {
  std::ifstream in(path.c_str());
  std::string contents;
  in.seekg(0, std::ios::end);
  contents.resize(in.tellg());
  in.seekg(0, std::ios::beg);
  in.read(&contents[0], contents.size());
  in.close();
  return(contents);
}

// [[Rcpp::export]]
CharacterVector read_file_cpp3(std::string path) {

  using namespace std;

  char* map;
  struct stat file_info;

  int fd = open( path.c_str(), O_RDONLY );
  if (fstat(fd, &file_info) == -1) {
    stop("Could not read file information.");
  }

  int sz = file_info.st_size;

  map = (char*) mmap(0, sz, PROT_READ, MAP_SHARED, fd, 0);

  if (map == MAP_FAILED) {
    close(fd);
    stop("Error mapping the file.");
  }

  CharacterVector output = no_init(1);
  SET_STRING_ELT(output, 0, Rf_mkCharLen(map, sz));

  munmap(map, sz);
  close(fd);

  return output;

}

## read-file.md

      
    Raw
  

              read-file.md
            
          
    Reading a complete file with R

This is a short exploration of the most efficient way to read a complete file
(including newlines) into R - previously I'd used readLines() plus paste()
but that's clearly the least efficient option.
Here are the options:


Use readLines() and paste()
read_file1 <- function(path) {
  paste0(paste0(readLines(path), collapse = "\n"), "\n")
}


Find out the size of the file and then use readChar()
read_file2 <- function(path) {
  size <- file.info(path)$size
  readChar(path, size, useBytes = TRUE)
}


As above, but using readBin(), then converting to a character vector.
Unfortunately you can't read into a character vector directly because
use type = "character" is limited to 10000 characters
read_file3 <- function(path) {
  size <- file.info(path)$size
  rawToChar(readBin(path, "raw", size))
}


A safer approach that doesn't use a separate call to file.info() - this avoids race conditions where the file changes between asking for its size and reading it. (Suggested by @klmr)
read_file4 <- function(path, chunk_size = 1e4) {
  con <- file(path, "rb", raw = TRUE)
  on.exit(close(con))
  
  # Guess approximate number of chunks
  n <- file.info(path)$size / chunk_size
  chunks <- vector("list", n)

  i <- 1L
  chunks[[i]] <- readBin(con, "raw", n = chunk_size)
  while(length(chunks[[i]]) == chunk_size) {
    i <- i + 1L
    chunks[[i]] <- readBin(con, "raw", n = chunk_size)
  }
  
  rawToChar(unlist(chunks, use.names = FALSE))
}


An alternative would be to use C++.  This version was supplied by @tim_yates
library(Rcpp)
sourceCpp("read-file.cpp")


An alternative would be to use C++.  read_file_cpp1 came from @tim_yates, and read_file_cpp2 from @the_belial
library(Rcpp)
sourceCpp("read-file.cpp")


We'll compare the results on a file included with R:
path <- file.path(R.home("doc"), "COPYING")
file.info(path)$size / 1024
# [1] 17.6

First we need to check they all return the same results. (They won't if the file
doesn't include a trailing newline)
stopifnot(identical(read_file1(path), read_file2(path)))
stopifnot(identical(read_file1(path), read_file3(path)))
stopifnot(identical(read_file1(path), read_file4(path)))
stopifnot(identical(read_file1(path), read_file_cpp1(path)))
stopifnot(identical(read_file1(path), read_file_cpp2(path)))
stopifnot(identical(read_file1(path), read_file_cpp3(path)))
The benchmarking results are clear: readChar() is the best base R option, and is
about four times faster for this file.  The safer approach using chunked readBin() reads is about 50% slower. The C++ functions both fast (2x faster than readChar() and 10x faster than readLines()) and safe.
microbenchmark( times=1000,
  readLines = read_file1(path),   
  readChar = read_file2(path),   
  readBin = read_file3(path),
  chunked_read = read_file4(path),
  Rcpp = read_file_cpp1(path),
  Rcpp2 = read_file_cpp2(path),
  Rcpp3 = read_file_cpp3(path)
)
# Unit: microseconds
#          expr    min     lq median   uq  max neval
#     readLines 2041.6 2118.9 2146.3 2222 3878  1000
#      readChar  154.6  168.6  184.9  227 1647  1000
#       readBin  172.4  187.9  208.0  263 1897  1000
#  chunked_read  234.7  255.5  274.3  328 2199  1000
#          Rcpp   94.0  104.1  113.5  156  403  1000
#         Rcpp2   75.8   84.2   91.9  136  328  1000
#         Rcpp3   79.7   86.7   95.3  136  453  1000


## read-file.rmd
```{r, echo = FALSE}
library(microbenchmark)
options(digits = 3)
opts_chunk$set(comment = "#", tidy = FALSE)
```

# Reading a complete file with R

This is a short exploration of the most efficient way to read a complete file
(including newlines) into R - previously I'd used `readLines()` plus `paste()`
but that's clearly the least efficient option.

Here are the options:

* Use `readLines()` and `paste()`

    ```{r}
    read_file1 <- function(path) {
      paste0(paste0(readLines(path), collapse = "\n"), "\n")
    }
    ```

* Find out the size of the file and then use `readChar()`

    ```{r}
    read_file2 <- function(path) {
      size <- file.info(path)$size
      readChar(path, size, useBytes = TRUE)
    }
    ```

* As above, but using `readBin()`, then converting to a character vector.
  Unfortunately you can't read into a character vector directly because
  use `type = "character"` is limited to 10000 characters

    ```{r}
    read_file3 <- function(path) {
      size <- file.info(path)$size
      rawToChar(readBin(path, "raw", size))
    }
    ```

* A safer approach that doesn't use a separate call to `file.info()` - this avoids race conditions where the file changes between asking for its size and reading it. (Suggested by [@klmr](http://twitter.com/klmr))

    ```{r}
    read_file4 <- function(path, chunk_size = 1e4) {
      con <- file(path, "rb", raw = TRUE)
      on.exit(close(con))

      # Guess approximate number of chunks
      n <- file.info(path)$size / chunk_size
      chunks <- vector("list", n)

      i <- 1L
      chunks[[i]] <- readBin(con, "raw", n = chunk_size)
      while(length(chunks[[i]]) == chunk_size) {
        i <- i + 1L
        chunks[[i]] <- readBin(con, "raw", n = chunk_size)
      }

      rawToChar(unlist(chunks, use.names = FALSE))
    }
    ```

* An alternative would be to use C++.  This version was supplied by [@tim_yates](http://twitter.com/tim_yates/status/372369074019258370)

    ```{r}
    library(Rcpp)
    sourceCpp("read-file.cpp")
    ```

* An alternative would be to use C++.  `read_file_cpp1` came from [@tim_yates](http://twitter.com/tim_yates/status/372369074019258370), and `read_file_cpp2` from [@the_belial](http://twitter.com/the_belal/status/372392150467489792)

    ```{r}
    library(Rcpp)
    sourceCpp("read-file.cpp")
    ```

We'll compare the results on a file included with R:
```{r}
path <- file.path(R.home("doc"), "COPYING")
file.info(path)$size / 1024
```

First we need to check they all return the same results. (They won't if the file
doesn't include a trailing newline)

```{r}
stopifnot(identical(read_file1(path), read_file2(path)))
stopifnot(identical(read_file1(path), read_file3(path)))
stopifnot(identical(read_file1(path), read_file4(path)))
stopifnot(identical(read_file1(path), read_file_cpp1(path)))
stopifnot(identical(read_file1(path), read_file_cpp2(path)))
stopifnot(identical(read_file1(path), read_file_cpp3(path)))
```

The benchmarking results are clear: `readChar()` is the best base R option, and is
about four times faster for this file.  The safer approach using chunked `readBin()` reads is about 50% slower. The C++ functions both fast (2x faster than `readChar()` and 10x faster than `readLines()`) and safe.

```{r}
microbenchmark( times=1000,
  readLines = read_file1(path),
  readChar = read_file2(path),
  readBin = read_file3(path),
  chunked_read = read_file4(path),
  Rcpp = read_file_cpp1(path),
  Rcpp2 = read_file_cpp2(path),
  Rcpp3 = read_file_cpp3(path)
)
```
	#include <fstream>
	#include <sstream>
	#include <string>
	#include <sys/mman.h>
	#include <sys/stat.h>
	#include <fcntl.h>
	#include <Rcpp.h>
	using namespace Rcpp;

	// [[Rcpp::export]]
	CharacterVector read_file_cpp1(std::string path) {
	std::ifstream t(path.c_str());
	std::stringstream ss;
	ss << t.rdbuf();
	return ss.str();
	}

	// [[Rcpp::export]]
	CharacterVector read_file_cpp2(std::string path) {
	std::ifstream in(path.c_str());
	std::string contents;
	in.seekg(0, std::ios::end);
	contents.resize(in.tellg());
	in.seekg(0, std::ios::beg);
	in.read(&contents[0], contents.size());
	in.close();
	return(contents);
	}

	// [[Rcpp::export]]
	CharacterVector read_file_cpp3(std::string path) {

	using namespace std;

	char* map;
	struct stat file_info;

	int fd = open( path.c_str(), O_RDONLY );
	if (fstat(fd, &file_info) == -1) {
	stop("Could not read file information.");
	}

	int sz = file_info.st_size;

	map = (char*) mmap(0, sz, PROT_READ, MAP_SHARED, fd, 0);

	if (map == MAP_FAILED) {
	close(fd);
	stop("Error mapping the file.");
	}

	CharacterVector output = no_init(1);
	SET_STRING_ELT(output, 0, Rf_mkCharLen(map, sz));

	munmap(map, sz);
	close(fd);

	return output;

	}
	```{r, echo = FALSE}
	library(microbenchmark)
	options(digits = 3)
	opts_chunk$set(comment = "#", tidy = FALSE)
	```

	# Reading a complete file with R

	This is a short exploration of the most efficient way to read a complete file
	(including newlines) into R - previously I'd used `readLines()` plus `paste()`
	but that's clearly the least efficient option.

	Here are the options:

	* Use `readLines()` and `paste()`

	```{r}
	read_file1 <- function(path) {
	paste0(paste0(readLines(path), collapse = "\n"), "\n")
	}
	```

	* Find out the size of the file and then use `readChar()`

	```{r}
	read_file2 <- function(path) {
	size <- file.info(path)$size
	readChar(path, size, useBytes = TRUE)
	}
	```

	* As above, but using `readBin()`, then converting to a character vector.
	Unfortunately you can't read into a character vector directly because
	use `type = "character"` is limited to 10000 characters

	```{r}
	read_file3 <- function(path) {
	size <- file.info(path)$size
	rawToChar(readBin(path, "raw", size))
	}
	```

	* A safer approach that doesn't use a separate call to `file.info()` - this avoids race conditions where the file changes between asking for its size and reading it. (Suggested by [@klmr](http://twitter.com/klmr))

	```{r}
	read_file4 <- function(path, chunk_size = 1e4) {
	con <- file(path, "rb", raw = TRUE)
	on.exit(close(con))

	# Guess approximate number of chunks
	n <- file.info(path)$size / chunk_size
	chunks <- vector("list", n)

	i <- 1L
	chunks[[i]] <- readBin(con, "raw", n = chunk_size)
	while(length(chunks[[i]]) == chunk_size) {
	i <- i + 1L
	chunks[[i]] <- readBin(con, "raw", n = chunk_size)
	}

	rawToChar(unlist(chunks, use.names = FALSE))
	}
	```

	* An alternative would be to use C++. This version was supplied by [@tim_yates](http://twitter.com/tim_yates/status/372369074019258370)

	```{r}
	library(Rcpp)
	sourceCpp("read-file.cpp")
	```

	* An alternative would be to use C++. `read_file_cpp1` came from [@tim_yates](http://twitter.com/tim_yates/status/372369074019258370), and `read_file_cpp2` from [@the_belial](http://twitter.com/the_belal/status/372392150467489792)

	```{r}
	library(Rcpp)
	sourceCpp("read-file.cpp")
	```

	We'll compare the results on a file included with R:
	```{r}
	path <- file.path(R.home("doc"), "COPYING")
	file.info(path)$size / 1024
	```

	First we need to check they all return the same results. (They won't if the file
	doesn't include a trailing newline)

	```{r}
	stopifnot(identical(read_file1(path), read_file2(path)))
	stopifnot(identical(read_file1(path), read_file3(path)))
	stopifnot(identical(read_file1(path), read_file4(path)))
	stopifnot(identical(read_file1(path), read_file_cpp1(path)))
	stopifnot(identical(read_file1(path), read_file_cpp2(path)))
	stopifnot(identical(read_file1(path), read_file_cpp3(path)))
	```

	The benchmarking results are clear: `readChar()` is the best base R option, and is
	about four times faster for this file. The safer approach using chunked `readBin()` reads is about 50% slower. The C++ functions both fast (2x faster than `readChar()` and 10x faster than `readLines()`) and safe.

	```{r}
	microbenchmark( times=1000,
	readLines = read_file1(path),
	readChar = read_file2(path),
	readBin = read_file3(path),
	chunked_read = read_file4(path),
	Rcpp = read_file_cpp1(path),
	Rcpp2 = read_file_cpp2(path),
	Rcpp3 = read_file_cpp3(path)
	)
	```