Skip to content

Instantly share code, notes, and snippets.

@hhatto
Created February 2, 2018 02:32
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save hhatto/a101f904e516c0ea8519cc5a50fcf586 to your computer and use it in GitHub Desktop.
Save hhatto/a101f904e516c0ea8519cc5a50fcf586 to your computer and use it in GitHub Desktop.
python csv module benchmark
from benchmarker import Benchmarker
import unicodecsv
import csv
import fcsv
NUM = 500 * 1000
NUM = 10 * 1000
source = [["abc", "def", "ghi"],
["jkl", "あいう", "opq"],
["vvv", "v1", "v2"]]
source = (("abc", "def", "ghi"),
("jkl", "あいう", "opq"),
("vvv", "v1", "v2"))
with Benchmarker(NUM, width=40) as bench:
@bench("std.writerows")
def b_std_writerows(bm):
for i in bm:
with open('b_std.csv', 'w') as out:
writer = csv.writer(out, quoting=csv.QUOTE_NONNUMERIC)
writer.writerows(source)
@bench("std.writerow")
def b_std_writerow(bm):
for i in bm:
with open('b_std.csv', 'w') as out:
writer = csv.writer(out, quoting=csv.QUOTE_NONNUMERIC)
writer.writerow(source[0])
writer.writerow(source[1])
writer.writerow(source[2])
@bench("unicodecsv.writerows")
def b_unicodecsv_writerows(bm):
for i in bm:
with open('b_unicodecsv.csv', 'wb') as out:
writer = unicodecsv.writer(out, quoting=csv.QUOTE_NONNUMERIC)
writer.writerows(source)
@bench("unicodecsv.writerow")
def b_unicodecsv_writerow(bm):
for i in bm:
with open('b_unicodecsv.csv', 'wb') as out:
writer = unicodecsv.writer(out, quoting=csv.QUOTE_NONNUMERIC)
writer.writerow(source[0])
writer.writerow(source[1])
writer.writerow(source[2])
@bench("fcsv.writerow")
def b_fcsv_writerow(bm):
for i in bm:
writer = fcsv.Writer('b_fcsv.csv') # , quoting=csv.QUOTE_NONNUMERIC)
writer.writerow(source[0])
writer.writerow(source[1])
writer.writerow(source[2])
@bench("fcsv.writerows")
def b_fcsv_writerows(bm):
for i in bm:
writer = fcsv.Writer('b_fcsv.csv') # , quoting=csv.QUOTE_NONNUMERIC)
writer.writerows(source)
with Benchmarker(NUM, width=40) as bench:
@bench("std.reader")
def b_std_reader(bm):
for i in bm:
with open('b_reader.csv') as fl:
reader = csv.reader(fl)
for row in reader:
_ = row
@bench("unicodecsv.reader")
def b_unicodecsv_reader(bm):
for i in bm:
with open('b_reader.csv', 'rb') as out:
reader = unicodecsv.reader(out, 'excel')
for row in reader:
_ = row
@bench("fcsv.reader")
def b_fcsv_reader(bm):
for i in bm:
reader = fcsv.Reader('b_reader.csv') # , quoting=csv.QUOTE_NONNUMERIC)
for row in reader.read():
_ = row
@hhatto
Copy link
Author

hhatto commented Feb 2, 2018

result:

## benchmarker:         release 4.0.1 (for python)
## python version:      3.6.4
## python compiler:     GCC 4.2.1 Compatible Apple LLVM 9.0.0 (clang-900.0.39.2)
## python platform:     Darwin-16.7.0-x86_64-i386-64bit
## python executable:   /Users/hattori-h/.virtualenvs/py3csvbench/bin/python
## cpu model:           Intel(R) Core(TM) i5-5257U CPU @ 2.70GHz
## parameters:          loop=10000, cycle=1, extra=0

##                                            real    (total    = user    + sys)
std.writerows                               3.1454    1.6700    0.6800    0.9900
std.writerow                                3.0796    1.6200    0.6800    0.9400
unicodecsv.writerows                        3.0286    1.5500    0.6300    0.9200
unicodecsv.writerow                         3.2753    1.6700    0.6700    1.0000
fcsv.writerow                               0.8595    0.6300    0.1800    0.4500
fcsv.writerows                              0.7215    0.6000    0.1800    0.4200

## Ranking                                    real
fcsv.writerows                              0.7215  (100.0) ********************
fcsv.writerow                               0.8595  ( 83.9) *****************
unicodecsv.writerows                        3.0286  ( 23.8) *****
std.writerow                                3.0796  ( 23.4) *****
std.writerows                               3.1454  ( 22.9) *****
unicodecsv.writerow                         3.2753  ( 22.0) ****

## Matrix                                     real    [01]    [02]    [03]    [04]    [05]    [06]
[01] fcsv.writerows                         0.7215   100.0   119.1   419.8   426.8   435.9   453.9
[02] fcsv.writerow                          0.8595    83.9   100.0   352.4   358.3   366.0   381.1
[03] unicodecsv.writerows                   3.0286    23.8    28.4   100.0   101.7   103.9   108.1
[04] std.writerow                           3.0796    23.4    27.9    98.3   100.0   102.1   106.4
[05] std.writerows                          3.1454    22.9    27.3    96.3    97.9   100.0   104.1
[06] unicodecsv.writerow                    3.2753    22.0    26.2    92.5    94.0    96.0   100.0

## benchmarker:         release 4.0.1 (for python)
## python version:      3.6.4
## python compiler:     GCC 4.2.1 Compatible Apple LLVM 9.0.0 (clang-900.0.39.2)
## python platform:     Darwin-16.7.0-x86_64-i386-64bit
## python executable:   /Users/hattori-h/.virtualenvs/py3csvbench/bin/python
## cpu model:           Intel(R) Core(TM) i5-5257U CPU @ 2.70GHz
## parameters:          loop=10000, cycle=1, extra=0

##                                            real    (total    = user    + sys)
std.reader                                  0.7892    0.6800    0.5000    0.1800
unicodecsv.reader                           0.6474    0.5600    0.3800    0.1800
fcsv.reader                                 0.5444    0.5000    0.4100    0.0900

## Ranking                                    real
fcsv.reader                                 0.5444  (100.0) ********************
unicodecsv.reader                           0.6474  ( 84.1) *****************
std.reader                                  0.7892  ( 69.0) **************

## Matrix                                     real    [01]    [02]    [03]
[01] fcsv.reader                            0.5444   100.0   118.9   145.0
[02] unicodecsv.reader                      0.6474    84.1   100.0   121.9
[03] std.reader                             0.7892    69.0    82.0   100.0

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment