arunsrinivasan

Here's the code for comparing base R (v3.0.3 and 3.1.0 vs data.table's sub-assignment by reference feature):

require(data.table)
set.seed(20140430)
N <- as.integer(10^(3:7)*2L)
ans = vector("list", length(N))
for (i in seq_along(N)) {
    print(i)
    nreg = N[i]

arunsrinivasan

require(data.table)

DT <- as.data.table(mtcars)

# directly using .SD
set.seed(45L)
system.time(ans1 <- DT[, .SD[sample(.N, 5L)], by=gear])
#   user  system elapsed 
#  0.009   0.000   0.010 
  

arunsrinivasan

require(dplyr)
require(data.table)

foo <- function(N) {

    group_sizes = 10^(1:(log10(N)-1L))
    uniqval <- unique(runif(2*N))

    fans <- vector("list", length(group_sizes))
    for (i in seq_along(group_sizes)) {

arunsrinivasan

require(data.table)

set.seed(1L)
DT1 <- data.table(x=sample(1e7), y=as.numeric(sample(1e7)), z=sample(letters, 1e7, TRUE))
DT2 <- copy(DT1)

val <- runif(1e7)

# 'set' seems faster when adding 1-column
# =======================================

arunsrinivasan

# here's some sample data to test it out
require(data.table)
require(dplyr)
set.seed(45)
DF <- data.frame(x=sample(3, 25, TRUE), y=1:25, z=26:50)
DP <- tbl_df(DF) # for DPLYR data.frame object
DT <- data.table(DF)

# 1) row-wise subset (usually based on conditions):

arunsrinivasan

require(data.table)

# let's create data huge data.table
set.seed(1)
N <- 2e7 # size of DT
 
# generate a character vector of length about 1e5
foo <- function() paste(sample(letters, sample(5:9, 1), TRUE), collapse="")
ch <- replicate(1e5, foo())
ch <- unique(ch)

arunsrinivasan

# version 1.8.11 
require(data.table)
# Loading required package: data.table
# data.table 1.8.11  For help type: help("data.table")

## create a huge data.table:
## -------------------------
set.seed(1)
N <- 2e7 # size of DT

arunsrinivasan

# version 1.8.11 (commit 1048)
require(data.table)
# Loading required package: data.table
# data.table 1.8.11  For help type: help("data.table")

## create a huge data.table:
## -------------------------
set.seed(1)
N <- 2e7 # size of DT

arunsrinivasan

from pandas import *
from pandas.util.testing import rands
import random

N = 10000
ngroups = 10

def get_test_data(ngroups=100, n=N):
    unique_groups = range(ngroups)
    arr = np.asarray(np.tile(unique_groups, n / ngroups), dtype=object)

arunsrinivasan

require(data.table)
# tested on 1.8.10 AND 1.8.11, results don't differ much at all.

require(plyr)
# 1_8 version

set.seed(1000) # for reproducibility
N <- 1e4
foo <- function() paste(sample(letters, 10), collapse="")
indices <- replicate(N, foo())