Skip to content

Instantly share code, notes, and snippets.

@rklemme

rklemme/TestBufferingForCharIO.java

Last active August 29, 2015 14:01
Show Gist options
  • Save rklemme/fad399b5e7cc4d3b6d0c to your computer and use it in GitHub Desktop.
Save rklemme/fad399b5e7cc4d3b6d0c to your computer and use it in GitHub Desktop.
Download ZIP
Test program to investigate effects of buffering and buffer sizes for char based input from file.
Raw
output.txt
Start
Reading /tmp/tmp-char-ascii-4004479685424479203.txt bytes: 104857602 chars: 104857602
reader=SINGLE char= 0 byte= 0 3716927.000us
reader=SINGLE char= 0 byte= 1024 3685206.000us
reader=SINGLE char= 0 byte= 2048 3278808.000us
reader=SINGLE char= 0 byte= 8192 3560530.000us
reader=SINGLE char= 1024 byte= 0 809555.000us
reader=SINGLE char= 1024 byte= 1024 782035.000us
reader=SINGLE char= 1024 byte= 2048 1172425.000us
reader=SINGLE char= 1024 byte= 8192 1312639.000us
reader=SINGLE char= 2048 byte= 0 929860.000us
reader=SINGLE char= 2048 byte= 1024 1040306.000us
reader=SINGLE char= 2048 byte= 2048 1314209.000us
reader=SINGLE char= 2048 byte= 8192 1070333.000us
reader=SINGLE char= 8192 byte= 0 1442043.000us
reader=SINGLE char= 8192 byte= 1024 808655.000us
reader=SINGLE char= 8192 byte= 2048 813564.000us
reader=SINGLE char= 8192 byte= 8192 1147072.000us
reader=LINE char= 0 byte= 0 373626.000us
reader=LINE char= 0 byte= 1024 340572.000us
reader=LINE char= 0 byte= 2048 345767.000us
reader=LINE char= 0 byte= 8192 346670.000us
reader=LINE char= 1024 byte= 0 341090.000us
reader=LINE char= 1024 byte= 1024 340530.000us
reader=LINE char= 1024 byte= 2048 374522.000us
reader=LINE char= 1024 byte= 8192 352931.000us
reader=LINE char= 2048 byte= 0 357639.000us
reader=LINE char= 2048 byte= 1024 333741.000us
reader=LINE char= 2048 byte= 2048 374869.000us
reader=LINE char= 2048 byte= 8192 352889.000us
reader=LINE char= 8192 byte= 0 324838.000us
reader=LINE char= 8192 byte= 1024 334371.000us
reader=LINE char= 8192 byte= 2048 391492.000us
reader=LINE char= 8192 byte= 8192 362879.000us
reader=BUFFERED char= 0 byte= 0 159865.000us
reader=BUFFERED char= 0 byte= 1024 159965.000us
reader=BUFFERED char= 0 byte= 2048 156932.000us
reader=BUFFERED char= 0 byte= 8192 158182.000us
reader=BUFFERED char= 1024 byte= 0 159310.000us
reader=BUFFERED char= 1024 byte= 1024 171722.000us
reader=BUFFERED char= 1024 byte= 2048 233843.000us
reader=BUFFERED char= 1024 byte= 8192 227361.000us
reader=BUFFERED char= 2048 byte= 0 259845.000us
reader=BUFFERED char= 2048 byte= 1024 280532.000us
reader=BUFFERED char= 2048 byte= 2048 279555.000us
reader=BUFFERED char= 2048 byte= 8192 264532.000us
reader=BUFFERED char= 8192 byte= 0 280157.000us
reader=BUFFERED char= 8192 byte= 1024 192594.000us
reader=BUFFERED char= 8192 byte= 2048 192070.000us
reader=BUFFERED char= 8192 byte= 8192 193734.000us
reader=BUFFERED64K char= 0 byte= 0 187646.000us
reader=BUFFERED64K char= 0 byte= 1024 188163.000us
reader=BUFFERED64K char= 0 byte= 2048 188192.000us
reader=BUFFERED64K char= 0 byte= 8192 188268.000us
reader=BUFFERED64K char= 1024 byte= 0 187956.000us
reader=BUFFERED64K char= 1024 byte= 1024 188154.000us
reader=BUFFERED64K char= 1024 byte= 2048 188242.000us
reader=BUFFERED64K char= 1024 byte= 8192 188245.000us
reader=BUFFERED64K char= 2048 byte= 0 187445.000us
reader=BUFFERED64K char= 2048 byte= 1024 188316.000us
reader=BUFFERED64K char= 2048 byte= 2048 188197.000us
reader=BUFFERED64K char= 2048 byte= 8192 187968.000us
reader=BUFFERED64K char= 8192 byte= 0 187751.000us
reader=BUFFERED64K char= 8192 byte= 1024 188160.000us
reader=BUFFERED64K char= 8192 byte= 2048 188005.000us
reader=BUFFERED64K char= 8192 byte= 8192 188349.000us
reader=NioCharReader char= 1024 byte= 1024 direct=false 263796.000us
reader=NioCharReader char= 1024 byte= 2048 direct=false 200440.000us
reader=NioCharReader char= 1024 byte= 8192 direct=false 160536.000us
reader=NioCharReader char= 2048 byte= 1024 direct=false 255369.000us
reader=NioCharReader char= 2048 byte= 2048 direct=false 201430.000us
reader=NioCharReader char= 2048 byte= 8192 direct=false 162975.000us
reader=NioCharReader char= 8192 byte= 1024 direct=false 254948.000us
reader=NioCharReader char= 8192 byte= 2048 direct=false 201614.000us
reader=NioCharReader char= 8192 byte= 8192 direct=false 162386.000us
reader=NioCharReader char= 1024 byte= 1024 direct=true 490805.000us
reader=NioCharReader char= 1024 byte= 2048 direct=true 436848.000us
reader=NioCharReader char= 1024 byte= 8192 direct=true 399712.000us
reader=NioCharReader char= 2048 byte= 1024 direct=true 486611.000us
reader=NioCharReader char= 2048 byte= 2048 direct=true 438617.000us
reader=NioCharReader char= 2048 byte= 8192 direct=true 401804.000us
reader=NioCharReader char= 8192 byte= 1024 direct=true 486368.000us
reader=NioCharReader char= 8192 byte= 2048 direct=true 439289.000us
reader=NioCharReader char= 8192 byte= 8192 direct=true 400586.000us
reader=NioLineReader char= 1024 byte= 1024 direct=false 924439.000us
reader=NioLineReader char= 1024 byte= 2048 direct=false 921108.000us
reader=NioLineReader char= 1024 byte= 8192 direct=false 915415.000us
reader=NioLineReader char= 2048 byte= 1024 direct=false 968312.000us
reader=NioLineReader char= 2048 byte= 2048 direct=false 928766.000us
reader=NioLineReader char= 2048 byte= 8192 direct=false 921730.000us
reader=NioLineReader char= 8192 byte= 1024 direct=false 942170.000us
reader=NioLineReader char= 8192 byte= 2048 direct=false 887546.000us
reader=NioLineReader char= 8192 byte= 8192 direct=false 861686.000us
reader=NioLineReader char= 1024 byte= 1024 direct=true 1251148.000us
reader=NioLineReader char= 1024 byte= 2048 direct=true 1247842.000us
reader=NioLineReader char= 1024 byte= 8192 direct=true 1203504.000us
reader=NioLineReader char= 2048 byte= 1024 direct=true 1242397.000us
reader=NioLineReader char= 2048 byte= 2048 direct=true 1061240.000us
reader=NioLineReader char= 2048 byte= 8192 direct=true 1154534.000us
reader=NioLineReader char= 8192 byte= 1024 direct=true 1142865.000us
reader=NioLineReader char= 8192 byte= 2048 direct=true 1129508.000us
reader=NioLineReader char= 8192 byte= 8192 direct=true 1167778.000us
Reading /tmp/tmp-char-utf-727134951993887351.txt bytes: 308402234 chars: 104857602
reader=SINGLE char= 0 byte= 0 6551863.000us
reader=SINGLE char= 0 byte= 1024 6749784.000us
reader=SINGLE char= 0 byte= 2048 6759489.000us
reader=SINGLE char= 0 byte= 8192 7159491.000us
reader=SINGLE char= 1024 byte= 0 3317198.000us
reader=SINGLE char= 1024 byte= 1024 3003561.000us
reader=SINGLE char= 1024 byte= 2048 3258627.000us
reader=SINGLE char= 1024 byte= 8192 3364398.000us
reader=SINGLE char= 2048 byte= 0 2977592.000us
reader=SINGLE char= 2048 byte= 1024 2998418.000us
reader=SINGLE char= 2048 byte= 2048 3155971.000us
reader=SINGLE char= 2048 byte= 8192 4832263.000us
reader=SINGLE char= 8192 byte= 0 3255589.000us
reader=SINGLE char= 8192 byte= 1024 3266768.000us
reader=SINGLE char= 8192 byte= 2048 3043318.000us
reader=SINGLE char= 8192 byte= 8192 3367848.000us
reader=LINE char= 0 byte= 0 2185056.000us
reader=LINE char= 0 byte= 1024 2133979.000us
reader=LINE char= 0 byte= 2048 2606057.000us
reader=LINE char= 0 byte= 8192 2271582.000us
reader=LINE char= 1024 byte= 0 2349179.000us
reader=LINE char= 1024 byte= 1024 2134017.000us
reader=LINE char= 1024 byte= 2048 2415891.000us
reader=LINE char= 1024 byte= 8192 2427238.000us
reader=LINE char= 2048 byte= 0 2123470.000us
reader=LINE char= 2048 byte= 1024 2607496.000us
reader=LINE char= 2048 byte= 2048 2180722.000us
reader=LINE char= 2048 byte= 8192 2422312.000us
reader=LINE char= 8192 byte= 0 2418166.000us
reader=LINE char= 8192 byte= 1024 2766880.000us
reader=LINE char= 8192 byte= 2048 2289213.000us
reader=LINE char= 8192 byte= 8192 2366282.000us
reader=BUFFERED char= 0 byte= 0 1939557.000us
reader=BUFFERED char= 0 byte= 1024 1941291.000us
reader=BUFFERED char= 0 byte= 2048 1941830.000us
reader=BUFFERED char= 0 byte= 8192 2151863.000us
reader=BUFFERED char= 1024 byte= 0 1940264.000us
reader=BUFFERED char= 1024 byte= 1024 1943842.000us
reader=BUFFERED char= 1024 byte= 2048 1941973.000us
reader=BUFFERED char= 1024 byte= 8192 2082716.000us
reader=BUFFERED char= 2048 byte= 0 1974395.000us
reader=BUFFERED char= 2048 byte= 1024 1978193.000us
reader=BUFFERED char= 2048 byte= 2048 1976079.000us
reader=BUFFERED char= 2048 byte= 8192 2110660.000us
reader=BUFFERED char= 8192 byte= 0 1974551.000us
reader=BUFFERED char= 8192 byte= 1024 1976740.000us
reader=BUFFERED char= 8192 byte= 2048 1977695.000us
reader=BUFFERED char= 8192 byte= 8192 2109207.000us
reader=BUFFERED64K char= 0 byte= 0 1937762.000us
reader=BUFFERED64K char= 0 byte= 1024 1938648.000us
reader=BUFFERED64K char= 0 byte= 2048 1937123.000us
reader=BUFFERED64K char= 0 byte= 8192 2072527.000us
reader=BUFFERED64K char= 1024 byte= 0 1938306.000us
reader=BUFFERED64K char= 1024 byte= 1024 1938458.000us
reader=BUFFERED64K char= 1024 byte= 2048 1939081.000us
reader=BUFFERED64K char= 1024 byte= 8192 2072575.000us
reader=BUFFERED64K char= 2048 byte= 0 1937438.000us
reader=BUFFERED64K char= 2048 byte= 1024 1938456.000us
reader=BUFFERED64K char= 2048 byte= 2048 1938324.000us
reader=BUFFERED64K char= 2048 byte= 8192 2072919.000us
reader=BUFFERED64K char= 8192 byte= 0 1937550.000us
reader=BUFFERED64K char= 8192 byte= 1024 1938823.000us
reader=BUFFERED64K char= 8192 byte= 2048 1937946.000us
reader=BUFFERED64K char= 8192 byte= 8192 2072847.000us
reader=NioCharReader char= 1024 byte= 1024 direct=false 2289810.000us
reader=NioCharReader char= 1024 byte= 2048 direct=false 2098415.000us
reader=NioCharReader char= 1024 byte= 8192 direct=false 1970832.000us
reader=NioCharReader char= 2048 byte= 1024 direct=false 2285700.000us
reader=NioCharReader char= 2048 byte= 2048 direct=false 2098866.000us
reader=NioCharReader char= 2048 byte= 8192 direct=false 1970693.000us
reader=NioCharReader char= 8192 byte= 1024 direct=false 2288119.000us
reader=NioCharReader char= 8192 byte= 2048 direct=false 2098341.000us
reader=NioCharReader char= 8192 byte= 8192 direct=false 1967583.000us
reader=NioCharReader char= 1024 byte= 1024 direct=true 2903005.000us
reader=NioCharReader char= 1024 byte= 2048 direct=true 2741875.000us
reader=NioCharReader char= 1024 byte= 8192 direct=true 2618314.000us
reader=NioCharReader char= 2048 byte= 1024 direct=true 2901713.000us
reader=NioCharReader char= 2048 byte= 2048 direct=true 2744965.000us
reader=NioCharReader char= 2048 byte= 8192 direct=true 2620742.000us
reader=NioCharReader char= 8192 byte= 1024 direct=true 2902874.000us
reader=NioCharReader char= 8192 byte= 2048 direct=true 2753153.000us
reader=NioCharReader char= 8192 byte= 8192 direct=true 2618146.000us
reader=NioLineReader char= 1024 byte= 1024 direct=false 3075911.000us
reader=NioLineReader char= 1024 byte= 2048 direct=false 2933735.000us
reader=NioLineReader char= 1024 byte= 8192 direct=false 2663972.000us
reader=NioLineReader char= 2048 byte= 1024 direct=false 2937459.000us
reader=NioLineReader char= 2048 byte= 2048 direct=false 2711801.000us
reader=NioLineReader char= 2048 byte= 8192 direct=false 2588752.000us
reader=NioLineReader char= 8192 byte= 1024 direct=false 2989592.000us
reader=NioLineReader char= 8192 byte= 2048 direct=false 2883411.000us
reader=NioLineReader char= 8192 byte= 8192 direct=false 2686119.000us
reader=NioLineReader char= 1024 byte= 1024 direct=true 3553686.000us
reader=NioLineReader char= 1024 byte= 2048 direct=true 3676156.000us
reader=NioLineReader char= 1024 byte= 8192 direct=true 3583337.000us
reader=NioLineReader char= 2048 byte= 1024 direct=true 3771090.000us
reader=NioLineReader char= 2048 byte= 2048 direct=true 3384421.000us
reader=NioLineReader char= 2048 byte= 8192 direct=true 3357113.000us
reader=NioLineReader char= 8192 byte= 1024 direct=true 3829637.000us
reader=NioLineReader char= 8192 byte= 2048 direct=true 3617603.000us
reader=NioLineReader char= 8192 byte= 8192 direct=true 3350561.000us
Finish
Raw
output2.txt
Start
Reading /tmp/tmp-char-ascii-1536373161176149564.txt bytes: 104857602 chars: 104857602
reader=SINGLE char= 0 byte= 0 4309208.000us
reader=SINGLE char= 0 byte= 1024 4212608.000us
reader=SINGLE char= 0 byte= 2048 4084157.000us
reader=SINGLE char= 0 byte= 8192 3861968.000us
reader=SINGLE char= 1024 byte= 0 900993.000us
reader=SINGLE char= 1024 byte= 1024 891053.000us
reader=SINGLE char= 1024 byte= 2048 888447.000us
reader=SINGLE char= 1024 byte= 8192 904940.000us
reader=SINGLE char= 2048 byte= 0 888169.000us
reader=SINGLE char= 2048 byte= 1024 1267161.000us
reader=SINGLE char= 2048 byte= 2048 1459287.000us
reader=SINGLE char= 2048 byte= 8192 1934340.000us
reader=SINGLE char= 8192 byte= 0 1929112.000us
reader=SINGLE char= 8192 byte= 1024 1975659.000us
reader=SINGLE char= 8192 byte= 2048 1976461.000us
reader=SINGLE char= 8192 byte= 8192 1925895.000us
reader=LINE char= 0 byte= 0 604009.000us
reader=LINE char= 0 byte= 1024 592278.000us
reader=LINE char= 0 byte= 2048 561253.000us
reader=LINE char= 0 byte= 8192 593876.000us
reader=LINE char= 1024 byte= 0 599101.000us
reader=LINE char= 1024 byte= 1024 582607.000us
reader=LINE char= 1024 byte= 2048 566616.000us
reader=LINE char= 1024 byte= 8192 626960.000us
reader=LINE char= 2048 byte= 0 652301.000us
reader=LINE char= 2048 byte= 1024 623658.000us
reader=LINE char= 2048 byte= 2048 727884.000us
reader=LINE char= 2048 byte= 8192 544272.000us
reader=LINE char= 8192 byte= 0 686681.000us
reader=LINE char= 8192 byte= 1024 572193.000us
reader=LINE char= 8192 byte= 2048 659594.000us
reader=LINE char= 8192 byte= 8192 557222.000us
reader=BUFFERED char= 0 byte= 0 480135.000us
reader=BUFFERED char= 0 byte= 1024 466782.000us
reader=BUFFERED char= 0 byte= 2048 478772.000us
reader=BUFFERED char= 0 byte= 8192 464892.000us
reader=BUFFERED char= 1024 byte= 0 484146.000us
reader=BUFFERED char= 1024 byte= 1024 486030.000us
reader=BUFFERED char= 1024 byte= 2048 471020.000us
reader=BUFFERED char= 1024 byte= 8192 483952.000us
reader=BUFFERED char= 2048 byte= 0 504089.000us
reader=BUFFERED char= 2048 byte= 1024 503178.000us
reader=BUFFERED char= 2048 byte= 2048 497017.000us
reader=BUFFERED char= 2048 byte= 8192 501930.000us
reader=BUFFERED char= 8192 byte= 0 498995.000us
reader=BUFFERED char= 8192 byte= 1024 499851.000us
reader=BUFFERED char= 8192 byte= 2048 504605.000us
reader=BUFFERED char= 8192 byte= 8192 494743.000us
reader=BUFFERED64K char= 0 byte= 0 518512.000us
reader=BUFFERED64K char= 0 byte= 1024 519428.000us
reader=BUFFERED64K char= 0 byte= 2048 515154.000us
reader=BUFFERED64K char= 0 byte= 8192 515697.000us
reader=BUFFERED64K char= 1024 byte= 0 514617.000us
reader=BUFFERED64K char= 1024 byte= 1024 515713.000us
reader=BUFFERED64K char= 1024 byte= 2048 523133.000us
reader=BUFFERED64K char= 1024 byte= 8192 515543.000us
reader=BUFFERED64K char= 2048 byte= 0 515111.000us
reader=BUFFERED64K char= 2048 byte= 1024 515732.000us
reader=BUFFERED64K char= 2048 byte= 2048 515109.000us
reader=BUFFERED64K char= 2048 byte= 8192 515627.000us
reader=BUFFERED64K char= 8192 byte= 0 516117.000us
reader=BUFFERED64K char= 8192 byte= 1024 515470.000us
reader=BUFFERED64K char= 8192 byte= 2048 515019.000us
reader=BUFFERED64K char= 8192 byte= 8192 515165.000us
Reading /tmp/tmp-char-utf-8255120276968437831.txt bytes: 308402234 chars: 104857602
reader=SINGLE char= 0 byte= 0 6809385.000us
reader=SINGLE char= 0 byte= 1024 6504751.000us
reader=SINGLE char= 0 byte= 2048 6636271.000us
reader=SINGLE char= 0 byte= 8192 6649756.000us
reader=SINGLE char= 1024 byte= 0 3073988.000us
reader=SINGLE char= 1024 byte= 1024 3069266.000us
reader=SINGLE char= 1024 byte= 2048 3646530.000us
reader=SINGLE char= 1024 byte= 8192 3172781.000us
reader=SINGLE char= 2048 byte= 0 3030564.000us
reader=SINGLE char= 2048 byte= 1024 3065943.000us
reader=SINGLE char= 2048 byte= 2048 3054389.000us
reader=SINGLE char= 2048 byte= 8192 3412974.000us
reader=SINGLE char= 8192 byte= 0 3057997.000us
reader=SINGLE char= 8192 byte= 1024 3068520.000us
reader=SINGLE char= 8192 byte= 2048 3056691.000us
reader=SINGLE char= 8192 byte= 8192 3196040.000us
reader=LINE char= 0 byte= 0 2486703.000us
ERROR: expected: FileMeta [fileName=/tmp/tmp-char-utf-8255120276968437831.txt, charCount=104857602, charCrc=2265273366982090496]
got : FileMeta [fileName=null, charCount=104857583, charCrc=-4649463682814067712]
ERROR: expected: FileMeta [fileName=/tmp/tmp-char-utf-8255120276968437831.txt, charCount=104857602, charCrc=2265273366982090496]
got : FileMeta [fileName=null, charCount=104857583, charCrc=-4649463682814067712]
reader=LINE char= 0 byte= 1024 2416226.000us
ERROR: expected: FileMeta [fileName=/tmp/tmp-char-utf-8255120276968437831.txt, charCount=104857602, charCrc=2265273366982090496]
got : FileMeta [fileName=null, charCount=104857583, charCrc=-4649463682814067712]
reader=LINE char= 0 byte= 2048 2339396.000us
ERROR: expected: FileMeta [fileName=/tmp/tmp-char-utf-8255120276968437831.txt, charCount=104857602, charCrc=2265273366982090496]
got : FileMeta [fileName=null, charCount=104857583, charCrc=-4649463682814067712]
reader=LINE char= 0 byte= 8192 2533178.000us
ERROR: expected: FileMeta [fileName=/tmp/tmp-char-utf-8255120276968437831.txt, charCount=104857602, charCrc=2265273366982090496]
got : FileMeta [fileName=null, charCount=104857583, charCrc=-4649463682814067712]
reader=LINE char= 1024 byte= 0 2339283.000us
ERROR: expected: FileMeta [fileName=/tmp/tmp-char-utf-8255120276968437831.txt, charCount=104857602, charCrc=2265273366982090496]
got : FileMeta [fileName=null, charCount=104857583, charCrc=-4649463682814067712]
reader=LINE char= 1024 byte= 1024 2583537.000us
ERROR: expected: FileMeta [fileName=/tmp/tmp-char-utf-8255120276968437831.txt, charCount=104857602, charCrc=2265273366982090496]
got : FileMeta [fileName=null, charCount=104857583, charCrc=-4649463682814067712]
reader=LINE char= 1024 byte= 2048 2464098.000us
ERROR: expected: FileMeta [fileName=/tmp/tmp-char-utf-8255120276968437831.txt, charCount=104857602, charCrc=2265273366982090496]
got : FileMeta [fileName=null, charCount=104857583, charCrc=-4649463682814067712]
reader=LINE char= 1024 byte= 8192 2768545.000us
ERROR: expected: FileMeta [fileName=/tmp/tmp-char-utf-8255120276968437831.txt, charCount=104857602, charCrc=2265273366982090496]
got : FileMeta [fileName=null, charCount=104857583, charCrc=-4649463682814067712]
reader=LINE char= 2048 byte= 0 2647517.000us
ERROR: expected: FileMeta [fileName=/tmp/tmp-char-utf-8255120276968437831.txt, charCount=104857602, charCrc=2265273366982090496]
got : FileMeta [fileName=null, charCount=104857583, charCrc=-4649463682814067712]
reader=LINE char= 2048 byte= 1024 2390487.000us
ERROR: expected: FileMeta [fileName=/tmp/tmp-char-utf-8255120276968437831.txt, charCount=104857602, charCrc=2265273366982090496]
got : FileMeta [fileName=null, charCount=104857583, charCrc=-4649463682814067712]reader=
LINE char= 2048 byte= 2048 2771559.000us
ERROR: expected: FileMeta [fileName=/tmp/tmp-char-utf-8255120276968437831.txt, charCount=104857602, charCrc=2265273366982090496]
got : FileMeta [fileName=null, charCount=104857583, charCrc=-4649463682814067712]
reader=LINE char= 2048 byte= 8192 3024743.000us
ERROR: expected: FileMeta [fileName=/tmp/tmp-char-utf-8255120276968437831.txt, charCount=104857602, charCrc=2265273366982090496]
got : FileMeta [fileName=null, charCount=104857583, charCrc=-4649463682814067712]
reader=LINE char= 8192 byte= 0 2426423.000us
ERROR: expected: FileMeta [fileName=/tmp/tmp-char-utf-8255120276968437831.txt, charCount=104857602, charCrc=2265273366982090496]
got : FileMeta [fileName=null, charCount=104857583, charCrc=-4649463682814067712]
reader=LINE char= 8192 byte= 1024 2932872.000us
ERROR: expected: FileMeta [fileName=/tmp/tmp-char-utf-8255120276968437831.txt, charCount=104857602, charCrc=2265273366982090496]
got : FileMeta [fileName=null, charCount=104857583, charCrc=-4649463682814067712]
reader=LINE char= 8192 byte= 2048 2353860.000us
ERROR: expected: FileMeta [fileName=/tmp/tmp-char-utf-8255120276968437831.txt, charCount=104857602, charCrc=2265273366982090496]
got : FileMeta [fileName=null, charCount=104857583, charCrc=-4649463682814067712]
reader=LINE char= 8192 byte= 8192 2804264.000us
reader=BUFFERED char= 0 byte= 0 3151093.000us
reader=BUFFERED char= 0 byte= 1024 3145268.000us
reader=BUFFERED char= 0 byte= 2048 3147903.000us
reader=BUFFERED char= 0 byte= 8192 2300414.000us
reader=BUFFERED char= 1024 byte= 0 3141852.000us
reader=BUFFERED char= 1024 byte= 1024 3068794.000us
reader=BUFFERED char= 1024 byte= 2048 2981447.000us
reader=BUFFERED char= 1024 byte= 8192 3138823.000us
reader=BUFFERED char= 2048 byte= 0 3147216.000us
reader=BUFFERED char= 2048 byte= 1024 3172621.000us
reader=BUFFERED char= 2048 byte= 2048 3172594.000us
reader=BUFFERED char= 2048 byte= 8192 3354887.000us
reader=BUFFERED char= 8192 byte= 0 3163720.000us
reader=BUFFERED char= 8192 byte= 1024 3203037.000us
reader=BUFFERED char= 8192 byte= 2048 3166248.000us
reader=BUFFERED char= 8192 byte= 8192 3351748.000us
reader=BUFFERED64K char= 0 byte= 0 3128867.000us
reader=BUFFERED64K char= 0 byte= 1024 3131647.000us
reader=BUFFERED64K char= 0 byte= 2048 2870754.000us
reader=BUFFERED64K char= 0 byte= 8192 2465051.000us
reader=BUFFERED64K char= 1024 byte= 0 2111038.000us
reader=BUFFERED64K char= 1024 byte= 1024 3009588.000us
reader=BUFFERED64K char= 1024 byte= 2048 3133589.000us
reader=BUFFERED64K char= 1024 byte= 8192 3318486.000us
reader=BUFFERED64K char= 2048 byte= 0 2917318.000us
reader=BUFFERED64K char= 2048 byte= 1024 3135227.000us
reader=BUFFERED64K char= 2048 byte= 2048 3132798.000us
reader=BUFFERED64K char= 2048 byte= 8192 3320105.000us
reader=BUFFERED64K char= 8192 byte= 0 3127585.000us
reader=BUFFERED64K char= 8192 byte= 1024 2924861.000us
reader=BUFFERED64K char= 8192 byte= 2048 2776100.000us
reader=BUFFERED64K char= 8192 byte= 8192 3314881.000us
Reading /tmp/tmp-char-ascii-1536373161176149564.txt bytes: 104857602 chars: 104857602
reader=NioCharReader char= 1024 byte= 1024 direct=false 623960.000us
reader=NioCharReader char= 1024 byte= 1024 direct=true 813146.000us
reader=NioCharReader char= 1024 byte= 2048 direct=false 371722.000us
reader=NioCharReader char= 1024 byte= 2048 direct=true 593066.000us
reader=NioCharReader char= 1024 byte= 8192 direct=false 314906.000us
reader=NioCharReader char= 1024 byte= 8192 direct=true 563235.000us
reader=NioCharReader char= 2048 byte= 1024 direct=false 405766.000us
reader=NioCharReader char= 2048 byte= 1024 direct=true 872756.000us
reader=NioCharReader char= 2048 byte= 2048 direct=false 420242.000us
reader=NioCharReader char= 2048 byte= 2048 direct=true 898717.000us
reader=NioCharReader char= 2048 byte= 8192 direct=false 481413.000us
reader=NioCharReader char= 2048 byte= 8192 direct=true 837116.000us
reader=NioCharReader char= 8192 byte= 1024 direct=false 517692.000us
reader=NioCharReader char= 8192 byte= 1024 direct=true 763231.000us
reader=NioCharReader char= 8192 byte= 2048 direct=false 361425.000us
reader=NioCharReader char= 8192 byte= 2048 direct=true 602259.000us
reader=NioCharReader char= 8192 byte= 8192 direct=false 323609.000us
reader=NioCharReader char= 8192 byte= 8192 direct=true 563538.000us
reader=NioLineReader char= 1024 byte= 1024 direct=false 1167672.000us
reader=NioLineReader char= 1024 byte= 1024 direct=true 1273258.000us
reader=NioLineReader char= 1024 byte= 2048 direct=false 1091277.000us
reader=NioLineReader char= 1024 byte= 2048 direct=true 1188911.000us
reader=NioLineReader char= 1024 byte= 8192 direct=false 1036657.000us
reader=NioLineReader char= 1024 byte= 8192 direct=true 1207161.000us
reader=NioLineReader char= 2048 byte= 1024 direct=false 1127132.000us
reader=NioLineReader char= 2048 byte= 1024 direct=true 1281427.000us
reader=NioLineReader char= 2048 byte= 2048 direct=false 1197076.000us
reader=NioLineReader char= 2048 byte= 2048 direct=true 1392777.000us
reader=NioLineReader char= 2048 byte= 8192 direct=false 919724.000us
reader=NioLineReader char= 2048 byte= 8192 direct=true 1164638.000us
reader=NioLineReader char= 8192 byte= 1024 direct=false 1125513.000us
reader=NioLineReader char= 8192 byte= 1024 direct=true 1503200.000us
reader=NioLineReader char= 8192 byte= 2048 direct=false 970854.000us
reader=NioLineReader char= 8192 byte= 2048 direct=true 1390800.000us
reader=NioLineReader char= 8192 byte= 8192 direct=false 922418.000us
reader=NioLineReader char= 8192 byte= 8192 direct=true 1195013.000us
Reading /tmp/tmp-char-utf-8255120276968437831.txt bytes: 308402234 chars: 104857602
reader=NioCharReader char= 1024 byte= 1024 direct=false 2466832.000us
reader=NioCharReader char= 1024 byte= 1024 direct=true 3092318.000us
reader=NioCharReader char= 1024 byte= 2048 direct=false 2185235.000us
reader=NioCharReader char= 1024 byte= 2048 direct=true 3398682.000us
reader=NioCharReader char= 1024 byte= 8192 direct=false 2983615.000us
reader=NioCharReader char= 1024 byte= 8192 direct=true 2958349.000us
reader=NioCharReader char= 2048 byte= 1024 direct=false 2332864.000us
reader=NioCharReader char= 2048 byte= 1024 direct=true 3105869.000us
reader=NioCharReader char= 2048 byte= 2048 direct=false 2265091.000us
reader=NioCharReader char= 2048 byte= 2048 direct=true 2922959.000us
reader=NioCharReader char= 2048 byte= 8192 direct=false 2332361.000us
reader=NioCharReader char= 2048 byte= 8192 direct=true 4167623.000us
reader=NioCharReader char= 8192 byte= 1024 direct=false 3425569.000us
reader=NioCharReader char= 8192 byte= 1024 direct=true 4559130.000us
reader=NioCharReader char= 8192 byte= 2048 direct=false 3297374.000us
reader=NioCharReader char= 8192 byte= 2048 direct=true 4340816.000us
reader=NioCharReader char= 8192 byte= 8192 direct=false 3184622.000us
reader=NioCharReader char= 8192 byte= 8192 direct=true 4173856.000us
reader=NioLineReader char= 1024 byte= 1024 direct=false 3416056.000us
reader=NioLineReader char= 1024 byte= 1024 direct=true 3685802.000us
reader=NioLineReader char= 1024 byte= 2048 direct=false 2794199.000us
reader=NioLineReader char= 1024 byte= 2048 direct=true 5183371.000us
reader=NioLineReader char= 1024 byte= 8192 direct=false 3025162.000us
reader=NioLineReader char= 1024 byte= 8192 direct=true 4110376.000us
reader=NioLineReader char= 2048 byte= 1024 direct=false 2995525.000us
reader=NioLineReader char= 2048 byte= 1024 direct=true 5035473.000us
reader=NioLineReader char= 2048 byte= 2048 direct=false 2913143.000us
reader=NioLineReader char= 2048 byte= 2048 direct=true 3583224.000us
reader=NioLineReader char= 2048 byte= 8192 direct=false 2911121.000us
reader=NioLineReader char= 2048 byte= 8192 direct=true 4764181.000us
reader=NioLineReader char= 8192 byte= 1024 direct=false 2944323.000us
reader=NioLineReader char= 8192 byte= 1024 direct=true 4340211.000us
reader=NioLineReader char= 8192 byte= 2048 direct=false 2810150.000us
reader=NioLineReader char= 8192 byte= 2048 direct=true 5011093.000us
reader=NioLineReader char= 8192 byte= 8192 direct=false 2985626.000us
reader=NioLineReader char= 8192 byte= 8192 direct=true 4190362.000us
Finish
Raw
TestBufferingForCharIO.java
package file;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.Reader;
import java.io.Writer;
import java.nio.ByteBuffer;
import java.nio.CharBuffer;
import java.nio.channels.FileChannel;
import java.nio.charset.Charset;
import java.nio.charset.CharsetDecoder;
import java.nio.charset.CharsetEncoder;
import java.nio.charset.CoderResult;
import java.nio.file.StandardOpenOption;
import java.util.Arrays;
import java.util.List;
import java.util.Random;
import java.util.concurrent.TimeUnit;
public final class TestBufferingForCharIO {
private static final char NL = '\n';
private static final long FILE_CHAR_LEN = 100L * 1024 * 1024;
private static final long LINE_LEN = 80L;
private static final Charset UTF8 = Charset.forName("UTF-8");
private static final CharsetEncoder UTF8_ENCODER = UTF8.newEncoder();
private static final int[] BUFFER_SIZES = { 0, 1024, 2048, 8192 };
private static final int[] NIO_BUFFER_SIZES = { 1024, 2048, 8192 };
private static final int REPEATS = 4;
private static class CharSource {
private final Random rnd = new Random(0);
private final char min;
private final char max;
public CharSource(char min, char max) {
if (min > max) {
throw new IllegalArgumentException();
}
this.min = min;
this.max = max;
}
public char nextChar() {
char ch;
do {
ch = (char) (min + rnd.nextInt(max - min + 1));
} while (!UTF8_ENCODER.canEncode(ch));
return ch;
}
}
private static final class FileMeta {
private final File fileName;
private final long charCount;
private final long charCrc;
public FileMeta(File fileName, long charCount, long charCrc) {
this.fileName = fileName;
this.charCount = charCount;
this.charCrc = charCrc;
}
public File getFileName() {
return fileName;
}
public long getCharCount() {
return charCount;
}
public long getCharCrc() {
return charCrc;
}
@Override
public String toString() {
final StringBuilder sb = new StringBuilder();
sb.append("FileMeta [fileName=");
sb.append(fileName);
sb.append(", charCount=");
sb.append(charCount);
sb.append(", charCrc=");
sb.append(charCrc);
sb.append("]");
return sb.toString();
}
}
private static void readBuffered(final Reader r, final int bufferSize) throws IOException {
final char[] buffer = new char[bufferSize];
for (int i; (i = r.read(buffer)) != -1;) {
for (int j = 0; j < i; ++j) {
@SuppressWarnings("unused")
final char ch = buffer[j]; // access array
}
}
}
private enum Rdr {
SINGLE {
@Override
public void readAll(Reader r) throws IOException {
while (r.read() != -1)
;
}
},
LINE {
@Override
public void readAll(Reader r) throws IOException {
@SuppressWarnings("resource")
final BufferedReader br = r instanceof BufferedReader ? (BufferedReader) r : new BufferedReader(r, 1024);
while (br.readLine() != null)
;
}
},
BUFFERED {
@Override
public void readAll(Reader r) throws IOException {
readBuffered(r, 1024);
}
},
BUFFERED64K {
@Override
public void readAll(Reader r) throws IOException {
readBuffered(r, 65536);
}
};
public abstract void readAll(Reader r) throws IOException;
}
private static abstract class NioReader {
public void begin() {
// NOP
}
public abstract void readChunk(CharBuffer b);
public void finish(CharBuffer b) {
// NOP
}
@Override
public String toString() {
return getClass().getSimpleName();
}
}
private static class NioCharReader extends NioReader {
@Override
public void readChunk(CharBuffer b) {
while (b.hasRemaining()) {
b.get();
}
}
};
private static class NioLineReader extends NioReader {
private final StringBuilder sb = new StringBuilder();
@Override
public void begin() {
sb.setLength(0);
}
@Override
public void finish(CharBuffer b) {
readChunk(b);
if (sb.length() > 0) {
line();
}
}
@Override
public void readChunk(CharBuffer b) {
while (b.hasRemaining()) {
final char ch = b.get();
if (ch == NL) {
line();
} else {
sb.append(ch);
}
}
}
private String line() {
final String s = sb.toString();
sb.setLength(0);
return s;
}
};
public static void main(String[] args) throws IOException {
System.out.println("Start");
final FileMeta[] filesToRead = { createTestFile("ascii", FILE_CHAR_LEN, new CharSource(' ', (char) 127)),
createTestFile("utf", FILE_CHAR_LEN, new CharSource(Character.MIN_VALUE, Character.MAX_VALUE)) };
for (final FileMeta fileToRead : filesToRead) {
fileToRead.getFileName().deleteOnExit();
}
final List<NioReader> nioRdrs = Arrays.asList(new NioCharReader(), new NioLineReader());
for (final FileMeta fileMeta : filesToRead) {
final File fileToRead = fileMeta.getFileName();
System.out.println("Reading " + fileToRead + " bytes: " + fileToRead.length() + " chars: "
+ fileMeta.getCharCount());
for (final Rdr r : Rdr.values()) {
for (final int charBuffer : BUFFER_SIZES) {
for (final int byteBuffer : BUFFER_SIZES) {
long t = Long.MAX_VALUE;
for (int i = 0; i < REPEATS; ++i) {
t = Math.min(t, readFile(fileMeta, byteBuffer, charBuffer, r));
}
System.out.printf("reader=%-14s char=%6d byte=%6d %15.3fus\n", r, charBuffer, byteBuffer, (double) t);
}
}
}
System.out.println();
for (final NioReader r : nioRdrs) {
for (final boolean direct : new boolean[] { false, true }) {
for (final int charBuffer : NIO_BUFFER_SIZES) {
for (final int byteBuffer : NIO_BUFFER_SIZES) {
long t = Long.MAX_VALUE;
for (int i = 0; i < REPEATS; ++i) {
t = Math.min(t, readFileNio(fileMeta, direct, byteBuffer, charBuffer, r));
}
System.out.printf("reader=%-14s char=%6d byte=%6d direct=%-6s %15.3fus\n", r, charBuffer, byteBuffer,
direct, (double) t);
}
}
}
}
System.out.println();
}
System.out.println("Finish");
}
private static FileMeta createTestFile(String label, final long roughCharCount, CharSource source) throws IOException {
final File f = File.createTempFile("tmp-char-" + label + "-", ".txt");
final Writer fw = new OutputStreamWriter(new BufferedOutputStream(new FileOutputStream(f), 4096), UTF8);
try {
long count = 0;
while (count < roughCharCount) {
final char ch = source.nextChar();
assert Character.isDefined(ch);
assert UTF8_ENCODER.canEncode(ch);
fw.append(ch);
if (++count % LINE_LEN == 0) {
fw.append(NL);
++count;
}
}
fw.append(NL);
++count;
fw.flush();
return new FileMeta(f, count, 0);
} finally {
fw.close();
}
}
@SuppressWarnings("resource")
private static long readFile(final FileMeta fileMeta, final int byteBufferSize, final int charBufferSize,
final Rdr reader) throws IOException {
final File f = fileMeta.getFileName();
final InputStream bytesIn = new FileInputStream(f);
final InputStream bufferedBytesIn = byteBufferSize == 0 ? bytesIn
: new BufferedInputStream(bytesIn, byteBufferSize);
final Reader charIn = new InputStreamReader(bufferedBytesIn, UTF8);
final Reader bufferedCharIn = charBufferSize == 0 ? charIn : new BufferedReader(charIn, charBufferSize);
try {
final long t1 = System.nanoTime();
reader.readAll(bufferedCharIn);
assert bufferedCharIn.read() == -1;
final long t2 = System.nanoTime();
return TimeUnit.NANOSECONDS.toMicros(t2 - t1);
} finally {
bufferedCharIn.close();
}
}
private static long readFileNio(final FileMeta fileMeta, final boolean direct, final int byteBuffer,
final int charBuffer, final NioReader r) throws IOException {
final File fileToRead = fileMeta.getFileName();
final FileChannel fc = FileChannel.open(fileToRead.toPath(), StandardOpenOption.READ);
try {
final ByteBuffer bb = direct ? ByteBuffer.allocateDirect(byteBuffer) : ByteBuffer.allocate(byteBuffer);
final CharBuffer cb = CharBuffer.allocate(charBuffer);
final CharsetDecoder decoder = UTF8.newDecoder();
final long t1 = System.nanoTime();
r.begin();
bb.flip();
while (fc.read((ByteBuffer) bb.compact()) != -1) {
bb.flip();
CoderResult cr = CoderResult.OVERFLOW;
while (cr == CoderResult.OVERFLOW && bb.hasRemaining()) {
cb.clear();
cr = decoder.decode(bb, cb, false);
cb.flip();
// processing
r.readChunk(cb);
assert !cb.hasRemaining();
}
if (cr.isError()) {
cr.throwException();
}
}
bb.flip();
cb.clear();
decoder.decode(bb, cb, true);
cb.flip();
r.finish(cb);
assert !cb.hasRemaining();
final long t2 = System.nanoTime();
return TimeUnit.NANOSECONDS.toMicros(t2 - t1);
} finally {
fc.close();
}
}
}
Raw
TestBufferingForCharIOWithCRC.java
package file;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.Reader;
import java.io.Writer;
import java.nio.ByteBuffer;
import java.nio.CharBuffer;
import java.nio.channels.FileChannel;
import java.nio.charset.Charset;
import java.nio.charset.CharsetDecoder;
import java.nio.charset.CharsetEncoder;
import java.nio.charset.CoderResult;
import java.nio.file.StandardOpenOption;
import java.util.Arrays;
import java.util.List;
import java.util.Random;
import java.util.concurrent.TimeUnit;
public final class TestBufferingForCharIOWithCRC {
private static final char NL = '\n';
private static final long FILE_CHAR_LEN = 100L * 1024 * 1024;
private static final long LINE_LEN = 80L;
private static final Charset UTF8 = Charset.forName("UTF-8");
private static final CharsetEncoder UTF8_ENCODER = UTF8.newEncoder();
private static final int[] BUFFER_SIZES = { 0, 1024, 2048, 8192 };
private static final int[] NIO_BUFFER_SIZES = { 1024, 2048, 8192 };
private static final int REPEATS = 1;
private static class CharSource {
private final Random rnd = new Random(0);
private final char min;
private final char max;
public CharSource(char min, char max) {
if (min > max) {
throw new IllegalArgumentException();
}
this.min = min;
this.max = max;
}
public char nextChar() {
char ch;
do {
ch = (char) (min + rnd.nextInt(max - min + 1));
} while (!UTF8_ENCODER.canEncode(ch));
return ch;
}
}
private static final class CharCRC {
private long count;
private long crc;
public CharCRC reset() {
crc = count = 0;
return this;
}
public char add(final char c) {
crc = (crc << 3) ^ (crc >> 11) ^ c;
++count;
return c;
}
public <C extends CharSequence> C add(final C cs) {
for (int i = 0; i < cs.length(); ++i) {
add(cs.charAt(i));
}
return cs;
}
public long getCount() {
return count;
}
public long getCrc() {
return crc;
}
public FileMeta createMeta() {
return createMeta(null);
}
public FileMeta createMeta(final File fileName) {
return new FileMeta(fileName, count, crc);
}
}
private static final class FileMeta {
private final File fileName;
private final long charCount;
private final long charCrc;
public FileMeta(File fileName, long charCount, long charCrc) {
this.fileName = fileName;
this.charCount = charCount;
this.charCrc = charCrc;
}
public File getFileName() {
return fileName;
}
public long getCharCount() {
return charCount;
}
public long getCharCrc() {
return charCrc;
}
@Override
public String toString() {
final StringBuilder sb = new StringBuilder();
sb.append("FileMeta [fileName=");
sb.append(fileName);
sb.append(", charCount=");
sb.append(charCount);
sb.append(", charCrc=");
sb.append(charCrc);
sb.append("]");
return sb.toString();
}
}
private static FileMeta readBuffered(final Reader r, final int bufferSize) throws IOException {
final CharCRC crc = new CharCRC();
final char[] buffer = new char[bufferSize];
for (int i; (i = r.read(buffer)) != -1;) {
for (int j = 0; j < i; ++j) {
crc.add(buffer[j]);
}
}
return crc.createMeta();
}
private enum Rdr {
SINGLE {
@Override
public FileMeta readAll(Reader r) throws IOException {
final CharCRC crc = new CharCRC();
for (int i; (i = r.read()) != -1;) {
crc.add((char) i);
}
return crc.createMeta();
}
},
LINE {
@Override
public FileMeta readAll(Reader r) throws IOException {
final CharCRC crc = new CharCRC();
@SuppressWarnings("resource")
final BufferedReader br = r instanceof BufferedReader ? (BufferedReader) r : new BufferedReader(r, 1024);
for (String s; (s = br.readLine()) != null;) {
crc.add(s);
crc.add(NL);
}
return crc.createMeta();
}
},
BUFFERED {
@Override
public FileMeta readAll(Reader r) throws IOException {
return readBuffered(r, 1024);
}
},
BUFFERED64K {
@Override
public FileMeta readAll(Reader r) throws IOException {
return readBuffered(r, 65536);
}
};
public abstract FileMeta readAll(Reader r) throws IOException;
}
private static abstract class NioReader {
protected final CharCRC crc = new CharCRC();
public void begin() {
crc.reset();
}
public abstract void readChunk(CharBuffer b);
public FileMeta finish(CharBuffer b) {
crc.add(b);
return new FileMeta(null, crc.getCount(), crc.getCrc());
}
@Override
public String toString() {
return getClass().getSimpleName();
}
}
private static class NioCharReader extends NioReader {
@Override
public void readChunk(CharBuffer b) {
while (b.hasRemaining()) {
crc.add(b.get());
}
}
};
private static class NioLineReader extends NioReader {
private final StringBuilder sb = new StringBuilder();
@Override
public void begin() {
super.begin();
sb.setLength(0);
}
@Override
public FileMeta finish(CharBuffer b) {
readChunk(b);
if (sb.length() > 0) {
line();
}
return crc.createMeta();
}
@Override
public void readChunk(CharBuffer b) {
while (b.hasRemaining()) {
final char ch = crc.add(b.get());
if (ch == NL) {
line();
} else {
sb.append(ch);
}
}
}
private String line() {
final String s = sb.toString();
sb.setLength(0);
return s;
}
};
public static void main(String[] args) throws IOException {
System.out.println("Start");
final FileMeta[] filesToRead = { createTestFile("ascii", FILE_CHAR_LEN, new CharSource(' ', (char) 127)),
createTestFile("utf", FILE_CHAR_LEN, new CharSource(Character.MIN_VALUE, Character.MAX_VALUE)) };
for (final FileMeta fileToRead : filesToRead) {
fileToRead.getFileName().deleteOnExit();
}
for (final FileMeta fileMeta : filesToRead) {
final File fileToRead = fileMeta.getFileName();
System.out.println("Reading " + fileToRead + " bytes: " + fileToRead.length() + " chars: "
+ fileMeta.getCharCount());
for (final Rdr r : Rdr.values()) {
for (final int charBuffer : BUFFER_SIZES) {
for (final int byteBuffer : BUFFER_SIZES) {
long t = Long.MAX_VALUE;
for (int i = 0; i < REPEATS; ++i) {
t = Math.min(t, readFile(fileMeta, byteBuffer, charBuffer, r));
}
System.out.printf("reader=%-14s char=%6d byte=%6d %15.3fus\n", r, charBuffer, byteBuffer, (double) t);
}
}
}
System.out.println();
}
final List<NioReader> nioRdrs = Arrays.asList(new NioCharReader(), new NioLineReader());
for (final FileMeta fileMeta : filesToRead) {
final File fileToRead = fileMeta.getFileName();
System.out.println("Reading " + fileToRead + " bytes: " + fileToRead.length() + " chars: "
+ fileMeta.getCharCount());
for (final NioReader r : nioRdrs) {
for (final int charBuffer : NIO_BUFFER_SIZES) {
for (final int byteBuffer : NIO_BUFFER_SIZES) {
for (final boolean direct : new boolean[] { false, true }) {
long t = Long.MAX_VALUE;
for (int i = 0; i < REPEATS; ++i) {
t = Math.min(t, readFileNio(fileMeta, direct, byteBuffer, charBuffer, r));
}
System.out.printf("reader=%-14s char=%6d byte=%6d direct=%-6s %15.3fus\n", r, charBuffer, byteBuffer,
direct, (double) t);
}
}
}
}
System.out.println();
}
System.out.println("Finish");
}
private static FileMeta createTestFile(String label, final long roughCharCount, CharSource source) throws IOException {
final File f = File.createTempFile("tmp-char-" + label + "-", ".txt");
final Writer fw = new OutputStreamWriter(new BufferedOutputStream(new FileOutputStream(f), 4096), UTF8);
try {
final CharCRC crc = new CharCRC();
while (crc.getCount() < roughCharCount) {
final char ch = source.nextChar();
assert Character.isDefined(ch);
assert UTF8_ENCODER.canEncode(ch);
fw.append(crc.add(ch));
if (crc.getCount() % LINE_LEN == 0) {
fw.append(crc.add(NL));
}
}
fw.append(crc.add(NL));
return crc.createMeta(f);
} finally {
fw.close();
}
}
@SuppressWarnings("resource")
private static long readFile(final FileMeta fileMeta, final int byteBufferSize, final int charBufferSize,
final Rdr reader) throws IOException {
final File f = fileMeta.getFileName();
final InputStream bytesIn = new FileInputStream(f);
final InputStream bufferedBytesIn = byteBufferSize == 0 ? bytesIn
: new BufferedInputStream(bytesIn, byteBufferSize);
final Reader charIn = new InputStreamReader(bufferedBytesIn, UTF8);
final Reader bufferedCharIn = charBufferSize == 0 ? charIn : new BufferedReader(charIn, charBufferSize);
try {
final long t1 = System.nanoTime();
final FileMeta readMeta = reader.readAll(bufferedCharIn);
assert bufferedCharIn.read() == -1;
final long t2 = System.nanoTime();
crcCheck(fileMeta, readMeta);
return TimeUnit.NANOSECONDS.toMicros(t2 - t1);
} finally {
bufferedCharIn.close();
}
}
private static long readFileNio(final FileMeta fileMeta, final boolean direct, final int byteBuffer,
final int charBuffer, final NioReader r) throws IOException {
final File fileToRead = fileMeta.getFileName();
final FileChannel fc = FileChannel.open(fileToRead.toPath(), StandardOpenOption.READ);
try {
final ByteBuffer bb = direct ? ByteBuffer.allocateDirect(byteBuffer) : ByteBuffer.allocate(byteBuffer);
final CharBuffer cb = CharBuffer.allocate(charBuffer);
final CharsetDecoder decoder = UTF8.newDecoder();
final long t1 = System.nanoTime();
r.begin();
bb.flip();
while (fc.read((ByteBuffer) bb.compact()) != -1) {
bb.flip();
CoderResult cr = CoderResult.OVERFLOW;
while (cr == CoderResult.OVERFLOW && bb.hasRemaining()) {
cb.clear();
cr = decoder.decode(bb, cb, false);
cb.flip();
// processing
r.readChunk(cb);
assert !cb.hasRemaining();
}
if (cr.isError()) {
cr.throwException();
}
}
bb.flip();
cb.clear();
decoder.decode(bb, cb, true);
cb.flip();
final FileMeta readMeta = r.finish(cb);
assert !cb.hasRemaining();
final long t2 = System.nanoTime();
crcCheck(fileMeta, readMeta);
return TimeUnit.NANOSECONDS.toMicros(t2 - t1);
} finally {
fc.close();
}
}
private static void crcCheck(final FileMeta fileMeta, final FileMeta readMeta) {
if (readMeta.getCharCrc() != fileMeta.getCharCrc()) {
System.err.println("ERROR:\texpected: " + fileMeta + "\n\tgot : " + readMeta);
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment