Skip to content

Instantly share code, notes, and snippets.

@netj

netj/restoring_variable_ids_benchmark.ipynb

Last active January 16, 2017 20:13
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save netj/20e00e1719684ce08f38edff519f575a to your computer and use it in GitHub Desktop.
Save netj/20e00e1719684ce08f38edff519f575a to your computer and use it in GitHub Desktop.
Download ZIP
Display the source blob
Display the rendered blob
Raw
restoring_variable_ids_benchmark.ipynb
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Restoring full Variable IDs with Partition IDs"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## In Perl"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Writing a.pl\n"
]
}
],
"source": [
"%%file a.pl\n",
"#!/usr/bin/perl -w\n",
"use strict;\n",
"my $SHARD_BASE = $ENV{pid} << 48;\n",
"while (<>) {\n",
" my ($vid, @rest) = split(\" \");\n",
" print join(\"\\t\", ($SHARD_BASE | $vid), @rest), \"\\n\";\n",
"}"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## In Python"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Writing a.py\n"
]
}
],
"source": [
"%%file a.py\n",
"#!/usr/bin/env python\n",
"from __future__ import print_function\n",
"import sys, os\n",
"SHARD_BASE = int(os.environ[\"pid\"]) << 48\n",
"for line in sys.stdin:\n",
" vid, cid, prob = line.split()\n",
" print(\"\\t\".join([str(int(vid) | SHARD_BASE), cid, prob]))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## In AWK"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Writing a.awk\n"
]
}
],
"source": [
"%%file a.awk\n",
"#!/usr/bin/env bash\n",
": ${pid:?}\n",
"exec awk --bignum -v SHARD_BASE=$(($pid << 48)) '\n",
" {printf \"%s\\t%s\\t%s\\n\", or(SHARD_BASE, $1), $2, $3}\n",
"'"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## In jq"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Writing a.jq\n"
]
}
],
"source": [
"%%file a.jq\n",
"#!/usr/bin/env bash\n",
"exec jq -R -r --argjson SHARD_BASE $(($pid << 48)) '\n",
" split(\" \") |\n",
" [ (.[0] | tonumber + $SHARD_BASE | tostring)\n",
" , .[1:][] ] | join(\"\\t\")\n",
"'"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Correctness"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"10414574138294272\t200\t300\r\n",
"10414574138294274\t200\t300\r\n",
"10414574138294276\t200\t300\r\n",
"10414574138294276\t200\t300\r\n",
"10414574138294276\t200\t300\r\n",
"10414574138294278\t200\t300\r\n",
"10414574138294280\t200\t300\r\n",
"10414574138294280\t200\t300\r\n",
"10414574138294280\t200\t300\r\n",
"10414574138294282\t200\t300\r\n"
]
}
],
"source": [
"!printf '%d 200 300\\n' $(seq 10) | pid=37 bash a.jq"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"10414574138294273\t200\t300\r\n",
"10414574138294274\t200\t300\r\n",
"10414574138294275\t200\t300\r\n",
"10414574138294276\t200\t300\r\n",
"10414574138294277\t200\t300\r\n",
"10414574138294278\t200\t300\r\n",
"10414574138294279\t200\t300\r\n",
"10414574138294280\t200\t300\r\n",
"10414574138294281\t200\t300\r\n",
"10414574138294282\t200\t300\r\n"
]
}
],
"source": [
"!printf '%d 200 300\\n' $(seq 10) | pid=37 perl a.pl"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"10414574138294273\t200\t300\r\n",
"10414574138294274\t200\t300\r\n",
"10414574138294275\t200\t300\r\n",
"10414574138294276\t200\t300\r\n",
"10414574138294277\t200\t300\r\n",
"10414574138294278\t200\t300\r\n",
"10414574138294279\t200\t300\r\n",
"10414574138294280\t200\t300\r\n",
"10414574138294281\t200\t300\r\n",
"10414574138294282\t200\t300\r\n"
]
}
],
"source": [
"!printf '%d 200 300\\n' $(seq 10) | pid=37 python a.py"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Run with synthetic data and measure time"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Overwriting try.sh\n"
]
}
],
"source": [
"%%file try.sh\n",
"#!/usr/bin/env bash\n",
"set -euo pipefail\n",
": ${n:?}\n",
"\n",
"for pid in 0 1 $(( (2 << (64-48-1)) - 1)); do\n",
" export pid\n",
" echo \"## pid=$pid ########################################\"\n",
" paste -d' ' <(seq $n) <(seq $n) <(yes 0.1234 2>/dev/null | head -$n) |\n",
" gtime -v \"$@\" | (head; tail)\n",
" echo \"####################################################\"\n",
"done"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"!chmod +x try.sh"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"env: n=3000000\n"
]
}
],
"source": [
"%env n=3000000"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {
"collapsed": false,
"scrolled": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"## pid=0 ########################################\n",
"1\t1\t0.1234\n",
"2\t2\t0.1234\n",
"3\t3\t0.1234\n",
"4\t4\t0.1234\n",
"5\t5\t0.1234\n",
"6\t6\t0.1234\n",
"7\t7\t0.1234\n",
"8\t8\t0.1234\n",
"9\t9\t0.1234\n",
"10\t10\t0.1234\n",
"\tCommand being timed: \"perl -w a.pl\"\n",
"\tUser time (seconds): 4.82\n",
"\tSystem time (seconds): 0.05\n",
"\tPercent of CPU this job got: 99%\n",
"\tElapsed (wall clock) time (h:mm:ss or m:ss): 0:04.91\n",
"\tAverage shared text size (kbytes): 0\n",
"\tAverage unshared data size (kbytes): 0\n",
"\tAverage stack size (kbytes): 0\n",
"\tAverage total size (kbytes): 0\n",
"\tMaximum resident set size (kbytes): 14237696\n",
"\tAverage resident set size (kbytes): 0\n",
"\tMajor (requiring I/O) page faults: 0\n",
"\tMinor (reclaiming a frame) page faults: 2519\n",
"\tVoluntary context switches: 0\n",
"\tInvoluntary context switches: 1926\n",
"\tSwaps: 0\n",
"\tFile system inputs: 0\n",
"\tFile system outputs: 0\n",
"\tSocket messages sent: 0\n",
"\tSocket messages received: 0\n",
"\tSignals delivered: 0\n",
"\tPage size (bytes): 4096\n",
"\tExit status: 0\n",
"2999991\t2999991\t0.1234\n",
"2999992\t2999992\t0.1234\n",
"2999993\t2999993\t0.1234\n",
"2999994\t2999994\t0.1234\n",
"2999995\t2999995\t0.1234\n",
"2999996\t2999996\t0.1234\n",
"2999997\t2999997\t0.1234\n",
"2999998\t2999998\t0.1234\n",
"2999999\t2999999\t0.1234\n",
"3000000\t3000000\t0.1234\n",
"####################################################\n",
"## pid=1 ########################################\n",
"281474976710657\t1\t0.1234\n",
"281474976710658\t2\t0.1234\n",
"281474976710659\t3\t0.1234\n",
"281474976710660\t4\t0.1234\n",
"281474976710661\t5\t0.1234\n",
"281474976710662\t6\t0.1234\n",
"281474976710663\t7\t0.1234\n",
"281474976710664\t8\t0.1234\n",
"281474976710665\t9\t0.1234\n",
"281474976710666\t10\t0.1234\n",
"\tCommand being timed: \"perl -w a.pl\"\n",
"\tUser time (seconds): 5.10\n",
"\tSystem time (seconds): 0.05\n",
"\tPercent of CPU this job got: 99%\n",
"\tElapsed (wall clock) time (h:mm:ss or m:ss): 0:05.17\n",
"\tAverage shared text size (kbytes): 0\n",
"\tAverage unshared data size (kbytes): 0\n",
"\tAverage stack size (kbytes): 0\n",
"\tAverage total size (kbytes): 0\n",
"\tMaximum resident set size (kbytes): 14188544\n",
"\tAverage resident set size (kbytes): 0\n",
"\tMajor (requiring I/O) page faults: 0\n",
"\tMinor (reclaiming a frame) page faults: 2512\n",
"\tVoluntary context switches: 0\n",
"\tInvoluntary context switches: 1791\n",
"\tSwaps: 0\n",
"\tFile system inputs: 0\n",
"\tFile system outputs: 0\n",
"\tSocket messages sent: 0\n",
"\tSocket messages received: 0\n",
"\tSignals delivered: 0\n",
"\tPage size (bytes): 4096\n",
"\tExit status: 0\n",
"281474979710647\t2999991\t0.1234\n",
"281474979710648\t2999992\t0.1234\n",
"281474979710649\t2999993\t0.1234\n",
"281474979710650\t2999994\t0.1234\n",
"281474979710651\t2999995\t0.1234\n",
"281474979710652\t2999996\t0.1234\n",
"281474979710653\t2999997\t0.1234\n",
"281474979710654\t2999998\t0.1234\n",
"281474979710655\t2999999\t0.1234\n",
"281474979710656\t3000000\t0.1234\n",
"####################################################\n",
"## pid=65535 ########################################\n",
"18446462598732840961\t1\t0.1234\n",
"18446462598732840962\t2\t0.1234\n",
"18446462598732840963\t3\t0.1234\n",
"18446462598732840964\t4\t0.1234\n",
"18446462598732840965\t5\t0.1234\n",
"18446462598732840966\t6\t0.1234\n",
"18446462598732840967\t7\t0.1234\n",
"18446462598732840968\t8\t0.1234\n",
"18446462598732840969\t9\t0.1234\n",
"18446462598732840970\t10\t0.1234\n",
"\tCommand being timed: \"perl -w a.pl\"\n",
"\tUser time (seconds): 5.01\n",
"\tSystem time (seconds): 0.05\n",
"\tPercent of CPU this job got: 99%\n",
"\tElapsed (wall clock) time (h:mm:ss or m:ss): 0:05.09\n",
"\tAverage shared text size (kbytes): 0\n",
"\tAverage unshared data size (kbytes): 0\n",
"\tAverage stack size (kbytes): 0\n",
"\tAverage total size (kbytes): 0\n",
"\tMaximum resident set size (kbytes): 14221312\n",
"\tAverage resident set size (kbytes): 0\n",
"\tMajor (requiring I/O) page faults: 0\n",
"\tMinor (reclaiming a frame) page faults: 2513\n",
"\tVoluntary context switches: 0\n",
"\tInvoluntary context switches: 2185\n",
"\tSwaps: 0\n",
"\tFile system inputs: 0\n",
"\tFile system outputs: 0\n",
"\tSocket messages sent: 0\n",
"\tSocket messages received: 0\n",
"\tSignals delivered: 0\n",
"\tPage size (bytes): 4096\n",
"\tExit status: 0\n",
"18446462598735840951\t2999991\t0.1234\n",
"18446462598735840952\t2999992\t0.1234\n",
"18446462598735840953\t2999993\t0.1234\n",
"18446462598735840954\t2999994\t0.1234\n",
"18446462598735840955\t2999995\t0.1234\n",
"18446462598735840956\t2999996\t0.1234\n",
"18446462598735840957\t2999997\t0.1234\n",
"18446462598735840958\t2999998\t0.1234\n",
"18446462598735840959\t2999999\t0.1234\n",
"18446462598735840960\t3000000\t0.1234\n",
"####################################################\n",
"Summary of my perl5 (revision 5 version 18 subversion 2) configuration:\n",
" \n",
" Platform:\n",
" osname=darwin, osvers=16.0, archname=darwin-thread-multi-2level\n",
" uname='darwin osx300.apple.com 16.0 darwin kernel version 15.0.0: wed apr 6 00:55:38 pdt 2016; root:xnu-3247.1.106.2.8~1development_x86_64 x86_64 '\n",
" config_args='-ds -e -Dprefix=/usr -Dccflags=-g -pipe -Dldflags= -Dman3ext=3pm -Duseithreads -Duseshrplib -Dinc_version_list=none -Dcc=cc'\n",
" hint=recommended, useposix=true, d_sigaction=define\n",
" useithreads=define, usemultiplicity=define\n",
" useperlio=define, d_sfio=undef, uselargefiles=define, usesocks=undef\n",
" use64bitint=define, use64bitall=define, uselongdouble=undef\n",
" usemymalloc=n, bincompat5005=undef\n",
" Compiler:\n",
" cc='cc', ccflags ='-arch x86_64 -arch i386 -g -pipe -fno-common -DPERL_DARWIN -fno-strict-aliasing -fstack-protector',\n",
" optimize='-Os',\n",
" cppflags='-g -pipe -fno-common -DPERL_DARWIN -fno-strict-aliasing -fstack-protector'\n",
" ccversion='', gccversion='4.2.1 Compatible Apple LLVM 8.0.0 (clang-800.0.34)', gccosandvers=''\n",
" intsize=4, longsize=8, ptrsize=8, doublesize=8, byteorder=12345678\n",
" d_longlong=define, longlongsize=8, d_longdbl=define, longdblsize=16\n",
" ivtype='long', ivsize=8, nvtype='double', nvsize=8, Off_t='off_t', lseeksize=8\n",
" alignbytes=8, prototype=define\n",
" Linker and Libraries:\n",
" ld='cc -mmacosx-version-min=10.12.2', ldflags ='-arch x86_64 -arch i386 -fstack-protector'\n",
" libpth=/usr/lib /usr/local/lib\n",
" libs= \n",
" perllibs=\n",
" libc=, so=dylib, useshrplib=true, libperl=libperl.dylib\n",
" gnulibc_version=''\n",
" Dynamic Linking:\n",
" dlsrc=dl_dlopen.xs, dlext=bundle, d_dlsymun=undef, ccdlflags=' '\n",
" cccdlflags=' ', lddlflags='-arch x86_64 -arch i386 -bundle -undefined dynamic_lookup -fstack-protector'\n",
"\n",
"\n",
"Characteristics of this binary (from libperl): \n",
" Compile-time options: HAS_TIMES MULTIPLICITY PERLIO_LAYERS\n",
" PERL_DONT_CREATE_GVSV\n",
" PERL_HASH_FUNC_ONE_AT_A_TIME_HARD\n",
" PERL_IMPLICIT_CONTEXT PERL_MALLOC_WRAP\n",
" PERL_PRESERVE_IVUV PERL_SAWAMPERSAND USE_64_BIT_ALL\n",
" USE_64_BIT_INT USE_ITHREADS USE_LARGE_FILES\n",
" USE_LOCALE USE_LOCALE_COLLATE USE_LOCALE_CTYPE\n",
" USE_LOCALE_NUMERIC USE_PERLIO USE_PERL_ATOF\n",
" USE_REENTRANT_API\n",
" Locally applied patches:\n",
"\t/Library/Perl/Updates/<version> comes before system perl directories\n",
"\tinstallprivlib and installarchlib points to the Updates directory\n",
" Built under darwin\n",
" Compiled at Jul 30 2016 16:59:44\n",
" @INC:\n",
" /Library/Perl/5.18/darwin-thread-multi-2level\n",
" /Library/Perl/5.18\n",
" /Network/Library/Perl/5.18/darwin-thread-multi-2level\n",
" /Network/Library/Perl/5.18\n",
" /Library/Perl/Updates/5.18.2/darwin-thread-multi-2level\n",
" /Library/Perl/Updates/5.18.2\n",
" /System/Library/Perl/5.18/darwin-thread-multi-2level\n",
" /System/Library/Perl/5.18\n",
" /System/Library/Perl/Extras/5.18/darwin-thread-multi-2level\n",
" /System/Library/Perl/Extras/5.18\n",
" .\n"
]
}
],
"source": [
"!./try.sh perl -w a.pl\n",
"!perl -V"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {
"collapsed": false,
"scrolled": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"## pid=0 ########################################\n",
"1\t1\t0.1234\n",
"2\t2\t0.1234\n",
"3\t3\t0.1234\n",
"4\t4\t0.1234\n",
"5\t5\t0.1234\n",
"6\t6\t0.1234\n",
"7\t7\t0.1234\n",
"8\t8\t0.1234\n",
"9\t9\t0.1234\n",
"10\t10\t0.1234\n",
"\tCommand being timed: \"python2.7 a.py\"\n",
"\tUser time (seconds): 8.74\n",
"\tSystem time (seconds): 0.06\n",
"\tPercent of CPU this job got: 99%\n",
"\tElapsed (wall clock) time (h:mm:ss or m:ss): 0:08.82\n",
"\tAverage shared text size (kbytes): 0\n",
"\tAverage unshared data size (kbytes): 0\n",
"\tAverage stack size (kbytes): 0\n",
"\tAverage total size (kbytes): 0\n",
"\tMaximum resident set size (kbytes): 35733504\n",
"\tAverage resident set size (kbytes): 0\n",
"\tMajor (requiring I/O) page faults: 0\n",
"\tMinor (reclaiming a frame) page faults: 2460\n",
"\tVoluntary context switches: 0\n",
"\tInvoluntary context switches: 3673\n",
"\tSwaps: 0\n",
"\tFile system inputs: 0\n",
"\tFile system outputs: 0\n",
"\tSocket messages sent: 0\n",
"\tSocket messages received: 0\n",
"\tSignals delivered: 0\n",
"\tPage size (bytes): 4096\n",
"\tExit status: 0\n",
"2999991\t2999991\t0.1234\n",
"2999992\t2999992\t0.1234\n",
"2999993\t2999993\t0.1234\n",
"2999994\t2999994\t0.1234\n",
"2999995\t2999995\t0.1234\n",
"2999996\t2999996\t0.1234\n",
"2999997\t2999997\t0.1234\n",
"2999998\t2999998\t0.1234\n",
"2999999\t2999999\t0.1234\n",
"3000000\t3000000\t0.1234\n",
"####################################################\n",
"## pid=1 ########################################\n",
"281474976710657\t1\t0.1234\n",
"281474976710658\t2\t0.1234\n",
"281474976710659\t3\t0.1234\n",
"281474976710660\t4\t0.1234\n",
"281474976710661\t5\t0.1234\n",
"281474976710662\t6\t0.1234\n",
"281474976710663\t7\t0.1234\n",
"281474976710664\t8\t0.1234\n",
"281474976710665\t9\t0.1234\n",
"281474976710666\t10\t0.1234\n",
"\tCommand being timed: \"python2.7 a.py\"\n",
"\tUser time (seconds): 8.78\n",
"\tSystem time (seconds): 0.06\n",
"\tPercent of CPU this job got: 99%\n",
"\tElapsed (wall clock) time (h:mm:ss or m:ss): 0:08.88\n",
"\tAverage shared text size (kbytes): 0\n",
"\tAverage unshared data size (kbytes): 0\n",
"\tAverage stack size (kbytes): 0\n",
"\tAverage total size (kbytes): 0\n",
"\tMaximum resident set size (kbytes): 35831808\n",
"\tAverage resident set size (kbytes): 0\n",
"\tMajor (requiring I/O) page faults: 0\n",
"\tMinor (reclaiming a frame) page faults: 2477\n",
"\tVoluntary context switches: 0\n",
"\tInvoluntary context switches: 2875\n",
"\tSwaps: 0\n",
"\tFile system inputs: 0\n",
"\tFile system outputs: 0\n",
"\tSocket messages sent: 0\n",
"\tSocket messages received: 0\n",
"\tSignals delivered: 0\n",
"\tPage size (bytes): 4096\n",
"\tExit status: 0\n",
"281474979710647\t2999991\t0.1234\n",
"281474979710648\t2999992\t0.1234\n",
"281474979710649\t2999993\t0.1234\n",
"281474979710650\t2999994\t0.1234\n",
"281474979710651\t2999995\t0.1234\n",
"281474979710652\t2999996\t0.1234\n",
"281474979710653\t2999997\t0.1234\n",
"281474979710654\t2999998\t0.1234\n",
"281474979710655\t2999999\t0.1234\n",
"281474979710656\t3000000\t0.1234\n",
"####################################################\n",
"## pid=65535 ########################################\n",
"18446462598732840961\t1\t0.1234\n",
"18446462598732840962\t2\t0.1234\n",
"18446462598732840963\t3\t0.1234\n",
"18446462598732840964\t4\t0.1234\n",
"18446462598732840965\t5\t0.1234\n",
"18446462598732840966\t6\t0.1234\n",
"18446462598732840967\t7\t0.1234\n",
"18446462598732840968\t8\t0.1234\n",
"18446462598732840969\t9\t0.1234\n",
"18446462598732840970\t10\t0.1234\n",
"\tCommand being timed: \"python2.7 a.py\"\n",
"\tUser time (seconds): 9.31\n",
"\tSystem time (seconds): 0.06\n",
"\tPercent of CPU this job got: 99%\n",
"\tElapsed (wall clock) time (h:mm:ss or m:ss): 0:09.41\n",
"\tAverage shared text size (kbytes): 0\n",
"\tAverage unshared data size (kbytes): 0\n",
"\tAverage stack size (kbytes): 0\n",
"\tAverage total size (kbytes): 0\n",
"\tMaximum resident set size (kbytes): 35848192\n",
"\tAverage resident set size (kbytes): 0\n",
"\tMajor (requiring I/O) page faults: 0\n",
"\tMinor (reclaiming a frame) page faults: 2481\n",
"\tVoluntary context switches: 0\n",
"\tInvoluntary context switches: 3535\n",
"\tSwaps: 0\n",
"\tFile system inputs: 0\n",
"\tFile system outputs: 0\n",
"\tSocket messages sent: 0\n",
"\tSocket messages received: 0\n",
"\tSignals delivered: 0\n",
"\tPage size (bytes): 4096\n",
"\tExit status: 0\n",
"18446462598735840951\t2999991\t0.1234\n",
"18446462598735840952\t2999992\t0.1234\n",
"18446462598735840953\t2999993\t0.1234\n",
"18446462598735840954\t2999994\t0.1234\n",
"18446462598735840955\t2999995\t0.1234\n",
"18446462598735840956\t2999996\t0.1234\n",
"18446462598735840957\t2999997\t0.1234\n",
"18446462598735840958\t2999998\t0.1234\n",
"18446462598735840959\t2999999\t0.1234\n",
"18446462598735840960\t3000000\t0.1234\n",
"####################################################\n",
"2.7.10 (default, Dec 7 2015, 00:14:41) \n",
"[GCC 4.2.1 Compatible Apple LLVM 7.0.0 (clang-700.1.76)]\n"
]
}
],
"source": [
"!./try.sh python2.7 a.py\n",
"!python2.7 -c 'import sys; sys.stdout.write(sys.version + \"\\n\")'"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {
"collapsed": false,
"scrolled": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"## pid=0 ########################################\n",
"1\t1\t0.1234\n",
"2\t2\t0.1234\n",
"3\t3\t0.1234\n",
"4\t4\t0.1234\n",
"5\t5\t0.1234\n",
"6\t6\t0.1234\n",
"7\t7\t0.1234\n",
"8\t8\t0.1234\n",
"9\t9\t0.1234\n",
"10\t10\t0.1234\n",
"\tCommand being timed: \"python3.5 a.py\"\n",
"\tUser time (seconds): 8.08\n",
"\tSystem time (seconds): 0.08\n",
"\tPercent of CPU this job got: 99%\n",
"\tElapsed (wall clock) time (h:mm:ss or m:ss): 0:08.20\n",
"\tAverage shared text size (kbytes): 0\n",
"\tAverage unshared data size (kbytes): 0\n",
"\tAverage stack size (kbytes): 0\n",
"\tAverage total size (kbytes): 0\n",
"\tMaximum resident set size (kbytes): 46825472\n",
"\tAverage resident set size (kbytes): 0\n",
"\tMajor (requiring I/O) page faults: 0\n",
"\tMinor (reclaiming a frame) page faults: 4625\n",
"\tVoluntary context switches: 23\n",
"\tInvoluntary context switches: 2834\n",
"\tSwaps: 0\n",
"\tFile system inputs: 23\n",
"\tFile system outputs: 0\n",
"\tSocket messages sent: 0\n",
"\tSocket messages received: 0\n",
"\tSignals delivered: 0\n",
"\tPage size (bytes): 4096\n",
"\tExit status: 0\n",
"2999991\t2999991\t0.1234\n",
"2999992\t2999992\t0.1234\n",
"2999993\t2999993\t0.1234\n",
"2999994\t2999994\t0.1234\n",
"2999995\t2999995\t0.1234\n",
"2999996\t2999996\t0.1234\n",
"2999997\t2999997\t0.1234\n",
"2999998\t2999998\t0.1234\n",
"2999999\t2999999\t0.1234\n",
"3000000\t3000000\t0.1234\n",
"####################################################\n",
"## pid=1 ########################################\n",
"281474976710657\t1\t0.1234\n",
"281474976710658\t2\t0.1234\n",
"281474976710659\t3\t0.1234\n",
"281474976710660\t4\t0.1234\n",
"281474976710661\t5\t0.1234\n",
"281474976710662\t6\t0.1234\n",
"281474976710663\t7\t0.1234\n",
"281474976710664\t8\t0.1234\n",
"281474976710665\t9\t0.1234\n",
"281474976710666\t10\t0.1234\n",
"\tCommand being timed: \"python3.5 a.py\"\n",
"\tUser time (seconds): 8.35\n",
"\tSystem time (seconds): 0.09\n",
"\tPercent of CPU this job got: 99%\n",
"\tElapsed (wall clock) time (h:mm:ss or m:ss): 0:08.48\n",
"\tAverage shared text size (kbytes): 0\n",
"\tAverage unshared data size (kbytes): 0\n",
"\tAverage stack size (kbytes): 0\n",
"\tAverage total size (kbytes): 0\n",
"\tMaximum resident set size (kbytes): 45023232\n",
"\tAverage resident set size (kbytes): 0\n",
"\tMajor (requiring I/O) page faults: 0\n",
"\tMinor (reclaiming a frame) page faults: 4514\n",
"\tVoluntary context switches: 0\n",
"\tInvoluntary context switches: 3932\n",
"\tSwaps: 0\n",
"\tFile system inputs: 0\n",
"\tFile system outputs: 0\n",
"\tSocket messages sent: 0\n",
"\tSocket messages received: 0\n",
"\tSignals delivered: 0\n",
"\tPage size (bytes): 4096\n",
"\tExit status: 0\n",
"281474979710647\t2999991\t0.1234\n",
"281474979710648\t2999992\t0.1234\n",
"281474979710649\t2999993\t0.1234\n",
"281474979710650\t2999994\t0.1234\n",
"281474979710651\t2999995\t0.1234\n",
"281474979710652\t2999996\t0.1234\n",
"281474979710653\t2999997\t0.1234\n",
"281474979710654\t2999998\t0.1234\n",
"281474979710655\t2999999\t0.1234\n",
"281474979710656\t3000000\t0.1234\n",
"####################################################\n",
"## pid=65535 ########################################\n",
"18446462598732840961\t1\t0.1234\n",
"18446462598732840962\t2\t0.1234\n",
"18446462598732840963\t3\t0.1234\n",
"18446462598732840964\t4\t0.1234\n",
"18446462598732840965\t5\t0.1234\n",
"18446462598732840966\t6\t0.1234\n",
"18446462598732840967\t7\t0.1234\n",
"18446462598732840968\t8\t0.1234\n",
"18446462598732840969\t9\t0.1234\n",
"18446462598732840970\t10\t0.1234\n",
"\tCommand being timed: \"python3.5 a.py\"\n",
"\tUser time (seconds): 8.26\n",
"\tSystem time (seconds): 0.09\n",
"\tPercent of CPU this job got: 99%\n",
"\tElapsed (wall clock) time (h:mm:ss or m:ss): 0:08.40\n",
"\tAverage shared text size (kbytes): 0\n",
"\tAverage unshared data size (kbytes): 0\n",
"\tAverage stack size (kbytes): 0\n",
"\tAverage total size (kbytes): 0\n",
"\tMaximum resident set size (kbytes): 45023232\n",
"\tAverage resident set size (kbytes): 0\n",
"\tMajor (requiring I/O) page faults: 0\n",
"\tMinor (reclaiming a frame) page faults: 4517\n",
"\tVoluntary context switches: 0\n",
"\tInvoluntary context switches: 2795\n",
"\tSwaps: 0\n",
"\tFile system inputs: 0\n",
"\tFile system outputs: 0\n",
"\tSocket messages sent: 0\n",
"\tSocket messages received: 0\n",
"\tSignals delivered: 0\n",
"\tPage size (bytes): 4096\n",
"\tExit status: 0\n",
"18446462598735840951\t2999991\t0.1234\n",
"18446462598735840952\t2999992\t0.1234\n",
"18446462598735840953\t2999993\t0.1234\n",
"18446462598735840954\t2999994\t0.1234\n",
"18446462598735840955\t2999995\t0.1234\n",
"18446462598735840956\t2999996\t0.1234\n",
"18446462598735840957\t2999997\t0.1234\n",
"18446462598735840958\t2999998\t0.1234\n",
"18446462598735840959\t2999999\t0.1234\n",
"18446462598735840960\t3000000\t0.1234\n",
"####################################################\n",
"3.5.2 (default, Oct 29 2016, 09:23:44) \n",
"[GCC 4.2.1 Compatible Apple LLVM 8.0.0 (clang-800.0.42.1)]\n"
]
}
],
"source": [
"!./try.sh python3.5 a.py\n",
"!python3.5 -c 'import sys; sys.stdout.write(sys.version + \"\\n\")'"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {
"collapsed": false,
"scrolled": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"## pid=0 ########################################\n",
"1\t1\t0.1234\n",
"2\t2\t0.1234\n",
"3\t3\t0.1234\n",
"4\t4\t0.1234\n",
"5\t5\t0.1234\n",
"6\t6\t0.1234\n",
"7\t7\t0.1234\n",
"8\t8\t0.1234\n",
"9\t9\t0.1234\n",
"10\t10\t0.1234\n",
"\tCommand being timed: \"bash a.awk\"\n",
"\tUser time (seconds): 10.95\n",
"\tSystem time (seconds): 0.05\n",
"\tPercent of CPU this job got: 99%\n",
"\tElapsed (wall clock) time (h:mm:ss or m:ss): 0:11.04\n",
"\tAverage shared text size (kbytes): 0\n",
"\tAverage unshared data size (kbytes): 0\n",
"\tAverage stack size (kbytes): 0\n",
"\tAverage total size (kbytes): 0\n",
"\tMaximum resident set size (kbytes): 11943936\n",
"\tAverage resident set size (kbytes): 0\n",
"\tMajor (requiring I/O) page faults: 0\n",
"\tMinor (reclaiming a frame) page faults: 1599\n",
"\tVoluntary context switches: 2\n",
"\tInvoluntary context switches: 3102\n",
"\tSwaps: 0\n",
"\tFile system inputs: 2\n",
"\tFile system outputs: 0\n",
"\tSocket messages sent: 0\n",
"\tSocket messages received: 0\n",
"\tSignals delivered: 0\n",
"\tPage size (bytes): 4096\n",
"\tExit status: 0\n",
"2999991\t2999991\t0.1234\n",
"2999992\t2999992\t0.1234\n",
"2999993\t2999993\t0.1234\n",
"2999994\t2999994\t0.1234\n",
"2999995\t2999995\t0.1234\n",
"2999996\t2999996\t0.1234\n",
"2999997\t2999997\t0.1234\n",
"2999998\t2999998\t0.1234\n",
"2999999\t2999999\t0.1234\n",
"3000000\t3000000\t0.1234\n",
"####################################################\n",
"## pid=1 ########################################\n",
"281474976710657\t1\t0.1234\n",
"281474976710658\t2\t0.1234\n",
"281474976710659\t3\t0.1234\n",
"281474976710660\t4\t0.1234\n",
"281474976710661\t5\t0.1234\n",
"281474976710662\t6\t0.1234\n",
"281474976710663\t7\t0.1234\n",
"281474976710664\t8\t0.1234\n",
"281474976710665\t9\t0.1234\n",
"281474976710666\t10\t0.1234\n",
"\tCommand being timed: \"bash a.awk\"\n",
"\tUser time (seconds): 11.13\n",
"\tSystem time (seconds): 0.08\n",
"\tPercent of CPU this job got: 97%\n",
"\tElapsed (wall clock) time (h:mm:ss or m:ss): 0:11.46\n",
"\tAverage shared text size (kbytes): 0\n",
"\tAverage unshared data size (kbytes): 0\n",
"\tAverage stack size (kbytes): 0\n",
"\tAverage total size (kbytes): 0\n",
"\tMaximum resident set size (kbytes): 11976704\n",
"\tAverage resident set size (kbytes): 0\n",
"\tMajor (requiring I/O) page faults: 0\n",
"\tMinor (reclaiming a frame) page faults: 1594\n",
"\tVoluntary context switches: 0\n",
"\tInvoluntary context switches: 5429\n",
"\tSwaps: 0\n",
"\tFile system inputs: 0\n",
"\tFile system outputs: 0\n",
"\tSocket messages sent: 0\n",
"\tSocket messages received: 0\n",
"\tSignals delivered: 0\n",
"\tPage size (bytes): 4096\n",
"\tExit status: 0\n",
"281474979710647\t2999991\t0.1234\n",
"281474979710648\t2999992\t0.1234\n",
"281474979710649\t2999993\t0.1234\n",
"281474979710650\t2999994\t0.1234\n",
"281474979710651\t2999995\t0.1234\n",
"281474979710652\t2999996\t0.1234\n",
"281474979710653\t2999997\t0.1234\n",
"281474979710654\t2999998\t0.1234\n",
"281474979710655\t2999999\t0.1234\n",
"281474979710656\t3000000\t0.1234\n",
"####################################################\n",
"## pid=65535 ########################################\n",
"-281474976710655\t1\t0.1234\n",
"-281474976710654\t2\t0.1234\n",
"-281474976710653\t3\t0.1234\n",
"-281474976710652\t4\t0.1234\n",
"-281474976710651\t5\t0.1234\n",
"-281474976710650\t6\t0.1234\n",
"-281474976710649\t7\t0.1234\n",
"-281474976710648\t8\t0.1234\n",
"-281474976710647\t9\t0.1234\n",
"-281474976710646\t10\t0.1234\n",
"\tCommand being timed: \"bash a.awk\"\n",
"\tUser time (seconds): 11.16\n",
"\tSystem time (seconds): 0.07\n",
"\tPercent of CPU this job got: 99%\n",
"\tElapsed (wall clock) time (h:mm:ss or m:ss): 0:11.27\n",
"\tAverage shared text size (kbytes): 0\n",
"\tAverage unshared data size (kbytes): 0\n",
"\tAverage stack size (kbytes): 0\n",
"\tAverage total size (kbytes): 0\n",
"\tMaximum resident set size (kbytes): 11943936\n",
"\tAverage resident set size (kbytes): 0\n",
"\tMajor (requiring I/O) page faults: 0\n",
"\tMinor (reclaiming a frame) page faults: 1592\n",
"\tVoluntary context switches: 0\n",
"\tInvoluntary context switches: 4749\n",
"\tSwaps: 0\n",
"\tFile system inputs: 0\n",
"\tFile system outputs: 0\n",
"\tSocket messages sent: 0\n",
"\tSocket messages received: 0\n",
"\tSignals delivered: 0\n",
"\tPage size (bytes): 4096\n",
"\tExit status: 0\n",
"-281474973710665\t2999991\t0.1234\n",
"-281474973710664\t2999992\t0.1234\n",
"-281474973710663\t2999993\t0.1234\n",
"-281474973710662\t2999994\t0.1234\n",
"-281474973710661\t2999995\t0.1234\n",
"-281474973710660\t2999996\t0.1234\n",
"-281474973710659\t2999997\t0.1234\n",
"-281474973710658\t2999998\t0.1234\n",
"-281474973710657\t2999999\t0.1234\n",
"-281474973710656\t3000000\t0.1234\n",
"####################################################\n",
"GNU Awk 4.1.4, API: 1.1 (GNU MPFR 3.1.5, GNU MP 6.1.1)\n",
"Copyright (C) 1989, 1991-2016 Free Software Foundation.\n",
"\n",
"This program is free software; you can redistribute it and/or modify\n",
"it under the terms of the GNU General Public License as published by\n",
"the Free Software Foundation; either version 3 of the License, or\n",
"(at your option) any later version.\n",
"\n",
"This program is distributed in the hope that it will be useful,\n",
"but WITHOUT ANY WARRANTY; without even the implied warranty of\n",
"MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n",
"GNU General Public License for more details.\n",
"\n",
"You should have received a copy of the GNU General Public License\n",
"along with this program. If not, see http://www.gnu.org/licenses/.\n"
]
}
],
"source": [
"!./try.sh bash a.awk\n",
"!awk --version"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {
"collapsed": false,
"scrolled": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"## pid=0 ########################################\n",
"1\t1\t0.1234\n",
"2\t2\t0.1234\n",
"3\t3\t0.1234\n",
"4\t4\t0.1234\n",
"5\t5\t0.1234\n",
"6\t6\t0.1234\n",
"7\t7\t0.1234\n",
"8\t8\t0.1234\n",
"9\t9\t0.1234\n",
"10\t10\t0.1234\n",
"\tCommand being timed: \"bash a.jq\"\n",
"\tUser time (seconds): 35.40\n",
"\tSystem time (seconds): 0.16\n",
"\tPercent of CPU this job got: 98%\n",
"\tElapsed (wall clock) time (h:mm:ss or m:ss): 0:36.08\n",
"\tAverage shared text size (kbytes): 0\n",
"\tAverage unshared data size (kbytes): 0\n",
"\tAverage stack size (kbytes): 0\n",
"\tAverage total size (kbytes): 0\n",
"\tMaximum resident set size (kbytes): 10076160\n",
"\tAverage resident set size (kbytes): 0\n",
"\tMajor (requiring I/O) page faults: 0\n",
"\tMinor (reclaiming a frame) page faults: 1486\n",
"\tVoluntary context switches: 0\n",
"\tInvoluntary context switches: 13005\n",
"\tSwaps: 0\n",
"\tFile system inputs: 0\n",
"\tFile system outputs: 0\n",
"\tSocket messages sent: 0\n",
"\tSocket messages received: 0\n",
"\tSignals delivered: 0\n",
"\tPage size (bytes): 4096\n",
"\tExit status: 0\n",
"2999991\t2999991\t0.1234\n",
"2999992\t2999992\t0.1234\n",
"2999993\t2999993\t0.1234\n",
"2999994\t2999994\t0.1234\n",
"2999995\t2999995\t0.1234\n",
"2999996\t2999996\t0.1234\n",
"2999997\t2999997\t0.1234\n",
"2999998\t2999998\t0.1234\n",
"2999999\t2999999\t0.1234\n",
"3000000\t3000000\t0.1234\n",
"####################################################\n",
"## pid=1 ########################################\n",
"281474976710657\t1\t0.1234\n",
"281474976710658\t2\t0.1234\n",
"281474976710659\t3\t0.1234\n",
"281474976710660\t4\t0.1234\n",
"281474976710661\t5\t0.1234\n",
"281474976710662\t6\t0.1234\n",
"281474976710663\t7\t0.1234\n",
"281474976710664\t8\t0.1234\n",
"281474976710665\t9\t0.1234\n",
"281474976710666\t10\t0.1234\n",
"\tCommand being timed: \"bash a.jq\"\n",
"\tUser time (seconds): 35.32\n",
"\tSystem time (seconds): 0.15\n",
"\tPercent of CPU this job got: 99%\n",
"\tElapsed (wall clock) time (h:mm:ss or m:ss): 0:35.56\n",
"\tAverage shared text size (kbytes): 0\n",
"\tAverage unshared data size (kbytes): 0\n",
"\tAverage stack size (kbytes): 0\n",
"\tAverage total size (kbytes): 0\n",
"\tMaximum resident set size (kbytes): 10207232\n",
"\tAverage resident set size (kbytes): 0\n",
"\tMajor (requiring I/O) page faults: 0\n",
"\tMinor (reclaiming a frame) page faults: 1481\n",
"\tVoluntary context switches: 0\n",
"\tInvoluntary context switches: 9652\n",
"\tSwaps: 0\n",
"\tFile system inputs: 0\n",
"\tFile system outputs: 0\n",
"\tSocket messages sent: 0\n",
"\tSocket messages received: 0\n",
"\tSignals delivered: 0\n",
"\tPage size (bytes): 4096\n",
"\tExit status: 0\n",
"281474979710647\t2999991\t0.1234\n",
"281474979710648\t2999992\t0.1234\n",
"281474979710649\t2999993\t0.1234\n",
"281474979710650\t2999994\t0.1234\n",
"281474979710651\t2999995\t0.1234\n",
"281474979710652\t2999996\t0.1234\n",
"281474979710653\t2999997\t0.1234\n",
"281474979710654\t2999998\t0.1234\n",
"281474979710655\t2999999\t0.1234\n",
"281474979710656\t3000000\t0.1234\n",
"####################################################\n",
"## pid=65535 ########################################\n",
"-281474976710655\t1\t0.1234\n",
"-281474976710654\t2\t0.1234\n",
"-281474976710653\t3\t0.1234\n",
"-281474976710652\t4\t0.1234\n",
"-281474976710651\t5\t0.1234\n",
"-281474976710650\t6\t0.1234\n",
"-281474976710649\t7\t0.1234\n",
"-281474976710648\t8\t0.1234\n",
"-281474976710647\t9\t0.1234\n",
"-281474976710646\t10\t0.1234\n",
"\tCommand being timed: \"bash a.jq\"\n",
"\tUser time (seconds): 35.65\n",
"\tSystem time (seconds): 0.18\n",
"\tPercent of CPU this job got: 99%\n",
"\tElapsed (wall clock) time (h:mm:ss or m:ss): 0:35.99\n",
"\tAverage shared text size (kbytes): 0\n",
"\tAverage unshared data size (kbytes): 0\n",
"\tAverage stack size (kbytes): 0\n",
"\tAverage total size (kbytes): 0\n",
"\tMaximum resident set size (kbytes): 10076160\n",
"\tAverage resident set size (kbytes): 0\n",
"\tMajor (requiring I/O) page faults: 0\n",
"\tMinor (reclaiming a frame) page faults: 1482\n",
"\tVoluntary context switches: 0\n",
"\tInvoluntary context switches: 14754\n",
"\tSwaps: 0\n",
"\tFile system inputs: 0\n",
"\tFile system outputs: 0\n",
"\tSocket messages sent: 0\n",
"\tSocket messages received: 0\n",
"\tSignals delivered: 0\n",
"\tPage size (bytes): 4096\n",
"\tExit status: 0\n",
"-281474973710665\t2999991\t0.1234\n",
"-281474973710664\t2999992\t0.1234\n",
"-281474973710663\t2999993\t0.1234\n",
"-281474973710662\t2999994\t0.1234\n",
"-281474973710661\t2999995\t0.1234\n",
"-281474973710660\t2999996\t0.1234\n",
"-281474973710659\t2999997\t0.1234\n",
"-281474973710658\t2999998\t0.1234\n",
"-281474973710657\t2999999\t0.1234\n",
"-281474973710656\t3000000\t0.1234\n",
"####################################################\n",
"jq-1.5\n"
]
}
],
"source": [
"!./try.sh bash a.jq\n",
"!jq --version"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 2",
"language": "python",
"name": "python2"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.10"
}
},
"nbformat": 4,
"nbformat_minor": 0
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment