timeIt function for benchmarking Julia code

timeIt.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "using DataFrames\n",
    "function timeIt(fs::Array{Function}, k)\n",
    "    nf=length(fs)\n",
    "    #Initialize and output checks\n",
    "    outs_init=[f() for f in fs]\n",
    "    times=fill(0.0, 3, k, nf)\n",
    "    for (i,f) in enumerate(fs)\n",
    "        for j=1:k\n",
    "            t=@timed f()\n",
    "            times[:,j,i]=[t[2:end]...]\n",
    "        end\n",
    "    end\n",
    "    out_last=[f() for f in fs]\n",
    "    out_same=nf>1 ? [NaN; [out_last[i]==out_last[1] for i=2:nf]] : [true]\n",
    "    out_consistent=[x==y for (x,y) in zip(outs_init, out_last)]\n",
    "    times[2,:,:]/=1e6\n",
    "\n",
    "    averages=mean(times,2)\n",
    "    averages_min=minimum(averages,3)\n",
    "    medians=median(times,2)\n",
    "    medians_min=minimum(medians,3)\n",
    "    e_notation(x)=@sprintf(\"%.02e\", x)\n",
    "    results=DataFrame(\n",
    "        Function=map(string, fs),\n",
    "        Consistent=out_consistent,\n",
    "        Same=out_same,\n",
    "        Avg_t=map(e_notation, vec(averages[1,:,:])),\n",
    "        Avg_t_X= nf>1 ? map(e_notation, vec(averages[1,:,:])./vec(averages_min[1,:,:])) : [NaN],\n",
    "        Med_t=map(e_notation, vec(medians[1,:,:])),\n",
    "        Med_t_X= nf>1 ? map(e_notation, vec(medians[1,:,:])./vec(medians_min[1,:,:])) : [NaN],\n",
    "        Avg_mem=map(e_notation, vec(averages[2,:,:])),\n",
    "        Avg_mem_X= nf>1 ? map(e_notation, vec(averages[2,:,:])./vec(averages_min[2,:,:])) : [NaN],\n",
    "        Med_mem=map(e_notation, vec(medians[2,:,:])),\n",
    "        Med_mem_X= nf>1 ? map(e_notation, vec(medians[2,:,:])./vec(medians_min[2,:,:])) : [NaN]\n",
    "    )\n",
    "    return results\n",
    "end\n",
    "timeIt(f::Function, k)=timeIt([f], k)\n",
    "timeIt([()->(), ()->1],5);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"data-frame\"><tr><th></th><th>Function</th><th>Consistent</th><th>Same</th><th>Avg_t</th><th>Avg_t_X</th><th>Med_t</th><th>Med_t_X</th><th>Avg_mem</th><th>Avg_mem_X</th><th>Med_mem</th><th>Med_mem_X</th></tr><tr><th>1</th><td>f1</td><td>false</td><td>NaN</td><td>1.02e-04</td><td>6.34e+00</td><td>1.01e-04</td><td>8.87e+00</td><td>8.00e-02</td><td>1.00e+00</td><td>8.00e-02</td><td>1.00e+00</td></tr><tr><th>2</th><td>f2</td><td>false</td><td>false</td><td>3.84e-04</td><td>2.40e+01</td><td>3.80e-04</td><td>3.35e+01</td><td>3.20e-01</td><td>4.00e+00</td><td>3.20e-01</td><td>4.00e+00</td></tr><tr><th>3</th><td>f3</td><td>true</td><td>false</td><td>1.60e-05</td><td>1.00e+00</td><td>1.13e-05</td><td>1.00e+00</td><td>8.00e-02</td><td>1.00e+00</td><td>8.00e-02</td><td>1.00e+00</td></tr><tr><th>4</th><td>f4</td><td>true</td><td>false</td><td>2.74e-05</td><td>1.71e+00</td><td>2.21e-05</td><td>1.95e+00</td><td>1.60e-01</td><td>2.00e+00</td><td>1.60e-01</td><td>2.00e+00</td></tr></table>"
      ],
      "text/plain": [
       "4x11 DataFrame\n",
       "| Row | Function | Consistent | Same  | Avg_t      | Avg_t_X    | Med_t      |\n",
       "|-----|----------|------------|-------|------------|------------|------------|\n",
       "| 1   | \"f1\"     | false      | NaN   | \"1.02e-04\" | \"6.34e+00\" | \"1.01e-04\" |\n",
       "| 2   | \"f2\"     | false      | false | \"3.84e-04\" | \"2.40e+01\" | \"3.80e-04\" |\n",
       "| 3   | \"f3\"     | true       | false | \"1.60e-05\" | \"1.00e+00\" | \"1.13e-05\" |\n",
       "| 4   | \"f4\"     | true       | false | \"2.74e-05\" | \"1.71e+00\" | \"2.21e-05\" |\n",
       "\n",
       "| Row | Med_t_X    | Avg_mem    | Avg_mem_X  | Med_mem    | Med_mem_X  |\n",
       "|-----|------------|------------|------------|------------|------------|\n",
       "| 1   | \"8.87e+00\" | \"8.00e-02\" | \"1.00e+00\" | \"8.00e-02\" | \"1.00e+00\" |\n",
       "| 2   | \"3.35e+01\" | \"3.20e-01\" | \"4.00e+00\" | \"3.20e-01\" | \"4.00e+00\" |\n",
       "| 3   | \"1.00e+00\" | \"8.00e-02\" | \"1.00e+00\" | \"8.00e-02\" | \"1.00e+00\" |\n",
       "| 4   | \"1.95e+00\" | \"1.60e-01\" | \"2.00e+00\" | \"1.60e-01\" | \"2.00e+00\" |"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "A=randn(100,100)\n",
    "f1()=randn(100,100)\n",
    "f2()=randn(200,200)\n",
    "f3()=A*1\n",
    "f4()=2.0*A*0.5\n",
    "timeIt([f1, f2, f3, f4],100)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Julia 0.3.9",
   "language": "julia",
   "name": "julia-0.3"
  },
  "language_info": {
   "name": "julia",
   "version": "0.3.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}

timeIt.jl

using DataFrames
function timeIt(fs::Array{Function}, k)
    nf=length(fs)
    #Initialize and output checks
    outs_init=[f() for f in fs]
    times=fill(0.0, 3, k, nf)
    for (i,f) in enumerate(fs)
        for j=1:k
            t=@timed f()
            times[:,j,i]=[t[2:end]...]
        end
    end
    out_last=[f() for f in fs]
    out_same=nf>1 ? [NaN; [out_last[i]==out_last[1] for i=2:nf]] : [true]
    out_consistent=[x==y for (x,y) in zip(outs_init, out_last)]
    times[2,:,:]/=1e6

    averages=mean(times,2)
    averages_min=minimum(averages,3)
    medians=median(times,2)
    medians_min=minimum(medians,3)
    e_notation(x)=@sprintf("%.02e", x)
    results=DataFrame(
        Function=map(string, fs),
        Consistent=out_consistent,
        Same=out_same,
        Avg_t=map(e_notation, vec(averages[1,:,:])),
        Avg_t_X= nf>1 ? map(e_notation, vec(averages[1,:,:])./vec(averages_min[1,:,:])) : [NaN],
        Med_t=map(e_notation, vec(medians[1,:,:])),
        Med_t_X= nf>1 ? map(e_notation, vec(medians[1,:,:])./vec(medians_min[1,:,:])) : [NaN],
        Avg_mem=map(e_notation, vec(averages[2,:,:])),
        Avg_mem_X= nf>1 ? map(e_notation, vec(averages[2,:,:])./vec(averages_min[2,:,:])) : [NaN],
        Med_mem=map(e_notation, vec(medians[2,:,:])),
        Med_mem_X= nf>1 ? map(e_notation, vec(medians[2,:,:])./vec(medians_min[2,:,:])) : [NaN]
    )
    return results
end
timeIt(f::Function, k)=timeIt([f], k)
timeIt([()->(), ()->1],5);