Created
July 13, 2012 07:22
-
-
Save yishenggudou/3103335 to your computer and use it in GitHub Desktop.
mongo 插入性能
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| ###已csv导入的性能 | |
| In [42]: %time dd=[qlogdb.insert({'date':'2012071399','hit':0,'miss':0,'contentlength':10000,'request':12121121,'consumer':'qiyi'}) for i in range(10000)] | |
| CPU times: user 0.80 s, sys: 0.33 s, total: 1.14 s | |
| Wall time: 1.70 s | |
| In [43]: %time dd=[qlogdb.insert({'date':'2012071399','hit':0,'miss':0,'contentlength':10000,'request':12121121,'consumer':'qiyi'}) for i in range(600000)] | |
| KeyboardInterrupt | |
| In [43]: del dd | |
| In [44]: %time dd=[qlogdb.insert({'date':'2012071399','hit':0,'miss':0,'contentlength':10000,'request':12121121,'consumer':'qiyi'}) for i in range(600000)] | |
| CPU times: user 49.24 s, sys: 19.11 s, total: 68.35 s | |
| Wall time: 75.84 s |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| 日志文件 | |
| [root@ work_temp]# time ssh root@00.00.00.00 'tail -F /data2/nginx-video/logs/*' > 00.00.00.00_data2_logs.log | |
| real 5m50.250s | |
| user 1m57.271s | |
| sys 0m18.348s | |
| [root@ work_temp]# du -h | |
| 3.6G . | |
| [root@ work_temp]# | |
| 转为csv | |
| [root@ work_temp]# time python tojson.py /data/qlog/work_temp/00.00.00.00_data2_logs.log > 00.00.00.00_data2_logs.csv | |
| real 2m55.644s | |
| user 2m49.190s | |
| sys 0m4.394s | |
| [root@ work_temp]# cat 00.00.00.00_data2_logs.log |wc -l | |
| 11225430 | |
| [root@ work_temp]# cat 00.00.00.00_data2_logs.csv |wc -l | |
| 10331316 | |
| [root@ work_temp]# | |
| [root@ bin]# grep HIT /data/qlog/work_temp/10.11.50.222_data2_logs.log|wc -l | |
| 10816964 | |
| [root@ bin]# grep HIT /data/qlog/work_temp/10.11.50.222_data2_logs.csv|wc -l | |
| 10330758 | |
| [root@ bin]# | |
| [root@ bin]# time ./mongoimport --host localhost --db qlog --collection realtimelogs --type csv --file /data/qlog/work_temp/10.11.50.222_data2_logs.csv --headerline --upsert -u qlog -p qlog | |
| connected to: localhost | |
| 6152853/656601021 0% | |
| 96800 32266/second | |
| 12351830/656601021 1% | |
| 194500 32416/second | |
| 18550012/656601021 2% | |
| 292000 32444/second | |
| 24836708/656601021 3% | |
| 391000 32583/second | |
| 31042618/656601021 4% | |
| 488700 32580/second | |
| 37221820/656601021 5% | |
| 586000 32555/second | |
| 43514743/656601021 6% | |
| 685000 32619/second | |
| 49728001/656601021 7% | |
| 782800 32616/second | |
| 55807138/656601021 8% | |
| 878500 32537/second | |
| 61975052/656601021 9% | |
| 975600 32520/second | |
| 68288384/656601021 10% | |
| 1074900 32572/second | |
| 74540539/656601021 11% | |
| 1173300 32591/second | |
| 80714270/656601021 12% | |
| 1270400 32574/second | |
| 87052824/656601021 13% | |
| 1370200 32623/second | |
| 93312939/656601021 14% | |
| 1468800 32640/second | |
| 97328370/656601021 14% | |
| 1532000 31916/second | |
| 103179718/656601021 15% | |
| 1624100 31845/second | |
| 109649911/656601021 16% | |
| 1726000 31962/second | |
| 116063834/656601021 17% | |
| 1827000 32052/second | |
| 122337304/656601021 18% | |
| 1925800 32096/second | |
| 128719546/656601021 19% | |
| 2026300 32163/second | |
| 134970586/656601021 20% | |
| 2124600 32190/second | |
| 141069363/656601021 21% | |
| 2220600 32182/second | |
| 147280833/656601021 22% | |
| 2318400 32200/second | |
| 153583346/656601021 23% | |
| 2417600 32234/second | |
| 159814664/656601021 24% | |
| 2515600 32251/second | |
| 166100442/656601021 25% | |
| 2614500 32277/second | |
| 172429727/656601021 26% | |
| 2714100 32310/second | |
| 178629758/656601021 27% | |
| 2811800 32319/second | |
| 184773037/656601021 28% | |
| 2908600 32317/second | |
| 190961603/656601021 29% | |
| 3006100 32323/second | |
| 197188198/656601021 30% | |
| 3104100 32334/second | |
| 203453197/656601021 30% | |
| 3202700 32350/second | |
| 209702343/656601021 31% | |
| 3301000 32362/second | |
| 216147921/656601021 32% | |
| 3402500 32404/second | |
| 220289226/656601021 33% | |
| 3467700 32108/second | |
| 226240674/656601021 34% | |
| 3561400 32084/second | |
| 232666429/656601021 35% | |
| 3662500 32127/second | |
| 239221369/656601021 36% | |
| 3765800 32186/second | |
| 245784869/656601021 37% | |
| 3869200 32243/second | |
| 252024940/656601021 38% | |
| 3967400 32255/second | |
| 258389466/656601021 39% | |
| 4067600 32282/second | |
| 264664967/656601021 40% | |
| 4166500 32298/second | |
| 270813309/656601021 41% | |
| 4263500 32299/second | |
| 277087940/656601021 42% | |
| 4362100 32311/second | |
| 282807051/656601021 43% | |
| 4452100 32261/second | |
| 287627444/656601021 43% | |
| 4528000 32113/second | |
| 292643565/656601021 44% | |
| 4606900 31992/second | |
| 297584717/656601021 45% | |
| 4684600 31868/second | |
| 301865892/656601021 45% | |
| 4751900 31679/second | |
| 306903155/656601021 46% | |
| 4831100 31575/second | |
| 311644234/656601021 47% | |
| 4905600 31446/second | |
| 316334614/656601021 48% | |
| 4979300 31316/second | |
| 321660489/656601021 48% | |
| 5063000 31253/second | |
| 325513588/656601021 49% | |
| 5123500 31051/second | |
| 328591754/656601021 50% | |
| 5171900 30785/second | |
| 332226283/656601021 50% | |
| 5229000 30578/second | |
| 337300809/656601021 51% | |
| 5308800 30510/second | |
| 342817945/656601021 52% | |
| 5395600 30483/second | |
| 348425238/656601021 53% | |
| 5483700 30465/second | |
| 353600547/656601021 53% | |
| 5565100 30410/second | |
| 358849448/656601021 54% | |
| 5647600 30363/second | |
| 364113065/656601021 55% | |
| 5730300 30319/second | |
| 369729849/656601021 56% | |
| 5818600 30305/second | |
| 375094361/656601021 57% | |
| 5903000 30271/second | |
| 380249298/656601021 57% | |
| 5984000 30222/second | |
| 385265195/656601021 58% | |
| 6063000 30164/second | |
| 391751438/656601021 59% | |
| 6165000 30220/second | |
| 398013918/656601021 60% | |
| 6263400 30257/second | |
| 404331157/656601021 61% | |
| 6362800 30299/second | |
| 410426571/656601021 62% | |
| 6458600 30322/second | |
| 416559881/656601021 63% | |
| 6555100 30347/second | |
| 422584461/656601021 64% | |
| 6649800 30364/second | |
| 424379207/656601021 64% | |
| 6677900 29945/second | |
| 424546820/656601021 64% | |
| 6680600 28920/second | |
| 428367553/656601021 65% | |
| 6740700 28806/second | |
| 434649919/656601021 66% | |
| 6839400 28858/second | |
| 440982950/656601021 67% | |
| 6939100 28912/second | |
| 447449994/656601021 68% | |
| 7040900 28974/second | |
| 453914897/656601021 69% | |
| 7142700 29035/second | |
| 460297368/656601021 70% | |
| 7243100 29088/second | |
| 466785725/656601021 71% | |
| 7345000 29146/second | |
| 473294720/656601021 72% | |
| 7447400 29205/second | |
| 479686664/656601021 73% | |
| 7548000 29255/second | |
| 485959503/656601021 74% | |
| 7646700 29297/second | |
| 492198418/656601021 74% | |
| 7744900 29336/second | |
| 498419591/656601021 75% | |
| 7842900 29374/second | |
| 504690217/656601021 76% | |
| 7941600 29413/second | |
| 510990581/656601021 77% | |
| 8040800 29453/second | |
| 517280399/656601021 78% | |
| 8139800 29492/second | |
| 523456065/656601021 79% | |
| 8237000 29523/second | |
| 529627798/656601021 80% | |
| 8334100 29553/second | |
| 535844824/656601021 81% | |
| 8432900 29589/second | |
| 540139357/656601021 82% | |
| 8500500 29515/second | |
| 546434745/656601021 83% | |
| 8599500 29551/second | |
| 552781837/656601021 84% | |
| 8700000 29591/second | |
| 559218150/656601021 85% | |
| 8801300 29634/second | |
| 565601294/656601021 86% | |
| 8901300 29671/second | |
| 571808204/656601021 87% | |
| 8998500 29698/second | |
| 578015808/656601021 88% | |
| 9096100 29725/second | |
| 584475338/656601021 89% | |
| 9197700 29766/second | |
| 590725367/656601021 89% | |
| 9296000 29794/second | |
| 596999246/656601021 90% | |
| 9394300 29823/second | |
| 603358210/656601021 91% | |
| 9493600 29854/second | |
| 609468550/656601021 92% | |
| 9589800 29874/second | |
| 615645914/656601021 93% | |
| 9687000 29898/second | |
| 621795076/656601021 94% | |
| 9783800 29919/second | |
| 628119248/656601021 95% | |
| 9883300 29949/second | |
| 634347675/656601021 96% | |
| 9981300 29973/second | |
| 640549045/656601021 97% | |
| 10078800 29996/second | |
| 646865331/656601021 98% | |
| 10178200 30024/second | |
| 652975500/656601021 99% | |
| 10274300 30041/second | |
| imported 10331316 objects | |
| real 5m43.816s | |
| user 1m48.384s | |
| sys 0m6.054s |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment