fastimage を同期的に処理してみるテスト

use-iter.py

import requests
from fastimage.detect import bytes_to_size
import os
import subprocess
import time


def test(url):
    stream = requests.get(url, stream=True)
    body = bytes()

    for line in stream.iter_content(chunk_size=24):
        body += line
        image_size, image_type = bytes_to_size(body)
        if image_size and image_type:
            return {"size": image_size, "type": image_type}
        if image_size is not None or image_type in  ['bmp', 'tif']:
            return


for _ in range(10):
    print(subprocess.check_output(["ps", "u", str(os.getpid())]).decode('UTF-8').rstrip())
    url = "https://s3-ap-northeast-1.amazonaws.com/df-monkey-preview/testdata/images/300x800.jpg"
    time.sleep(1)

use-requests-module.py

import requests
from fastimage.detect import bytes_to_size
import os
import subprocess
import time


def test(url):
    stream = requests.get(url, stream=True)
    body = bytes()
    chunk = stream.raw.read(24)
    while chunk and len(body) < 3000000:
        body += chunk
        image_size, image_type = bytes_to_size(body)
        if image_size and image_type:
            return image_size, image_type
        if image_size is not None or image_type in  ['bmp', 'tif']:
            return
        chunk = stream.raw.read(8)

for _ in range(10):
    print(subprocess.check_output(["ps", "u", str(os.getpid())]).decode('UTF-8').rstrip())
    url = "https://s3-ap-northeast-1.amazonaws.com/df-monkey-preview/testdata/images/300x800.jpg"
    print(test(url))
    time.sleep(1)

$ python use-requests-module.py
USER        PID  %CPU %MEM      VSZ    RSS   TT  STAT STARTED      TIME COMMAND
pokotyamu 86208  47.1  0.2  2432876  27176 s007  S+   11:15AM   0:00.36 python use-requests-module.py
((300, 800), 'jpg')
USER        PID  %CPU %MEM      VSZ    RSS   TT  STAT STARTED      TIME COMMAND
pokotyamu 86208   0.9  0.2  2443952  31308 s007  S+   11:15AM   0:00.39 python use-requests-module.py
((300, 800), 'jpg')
USER        PID  %CPU %MEM      VSZ    RSS   TT  STAT STARTED      TIME COMMAND
pokotyamu 86208   0.1  0.2  2444976  31944 s007  S+   11:15AM   0:00.41 python use-requests-module.py
((300, 800), 'jpg')
USER        PID  %CPU %MEM      VSZ    RSS   TT  STAT STARTED      TIME COMMAND
pokotyamu 86208   0.4  0.2  2444976  32084 s007  S+   11:15AM   0:00.43 python use-requests-module.py
((300, 800), 'jpg')
USER        PID  %CPU %MEM      VSZ    RSS   TT  STAT STARTED      TIME COMMAND
pokotyamu 86208   0.2  0.2  2444976  32388 s007  S+   11:15AM   0:00.46 python use-requests-module.py
((300, 800), 'jpg')
USER        PID  %CPU %MEM      VSZ    RSS   TT  STAT STARTED      TIME COMMAND
pokotyamu 86208   0.1  0.2  2443928  32364 s007  S+   11:15AM   0:00.49 python use-requests-module.py
((300, 800), 'jpg')
USER        PID  %CPU %MEM      VSZ    RSS   TT  STAT STARTED      TIME COMMAND
pokotyamu 86208   0.1  0.2  2443928  32624 s007  S+   11:15AM   0:00.51 python use-requests-module.py
((300, 800), 'jpg')
USER        PID  %CPU %MEM      VSZ    RSS   TT  STAT STARTED      TIME COMMAND
pokotyamu 86208   0.1  0.2  2443928  32628 s007  S+   11:15AM   0:00.54 python use-requests-module.py
((300, 800), 'jpg')
USER        PID  %CPU %MEM      VSZ    RSS   TT  STAT STARTED      TIME COMMAND
pokotyamu 86208   0.3  0.2  2443928  32712 s007  S+   11:15AM   0:00.56 python use-requests-module.py
((300, 800), 'jpg')
USER        PID  %CPU %MEM      VSZ    RSS   TT  STAT STARTED      TIME COMMAND
pokotyamu 86208   0.2  0.2  2443928  32716 s007  S+   11:15AM   0:00.59 python use-requests-module.py
((300, 800), 'jpg')

print 関数消したら、更にいい感じになった。

python use-requests-module.py
USER        PID  %CPU %MEM      VSZ    RSS   TT  STAT STARTED      TIME COMMAND
pokotyamu  4991  51.0  0.2  2432876  27148 s008  S+    2:34PM   0:00.36 python use-requests-module.py
USER        PID  %CPU %MEM      VSZ    RSS   TT  STAT STARTED      TIME COMMAND
pokotyamu  4991   1.8  0.2  2432876  27164 s008  S+    2:34PM   0:00.36 python use-requests-module.py
USER        PID  %CPU %MEM      VSZ    RSS   TT  STAT STARTED      TIME COMMAND
pokotyamu  4991   0.0  0.2  2432876  27164 s008  S+    2:34PM   0:00.36 python use-requests-module.py
USER        PID  %CPU %MEM      VSZ    RSS   TT  STAT STARTED      TIME COMMAND
pokotyamu  4991   0.0  0.2  2432876  27172 s008  S+    2:34PM   0:00.37 python use-requests-module.py
USER        PID  %CPU %MEM      VSZ    RSS   TT  STAT STARTED      TIME COMMAND
pokotyamu  4991   0.0  0.2  2432876  27172 s008  S+    2:34PM   0:00.37 python use-requests-module.py
USER        PID  %CPU %MEM      VSZ    RSS   TT  STAT STARTED      TIME COMMAND
pokotyamu  4991   0.4  0.2  2432876  27172 s008  S+    2:34PM   0:00.37 python use-requests-module.py
USER        PID  %CPU %MEM      VSZ    RSS   TT  STAT STARTED      TIME COMMAND
pokotyamu  4991   0.0  0.2  2432876  27172 s008  S+    2:34PM   0:00.38 python use-requests-module.py
USER        PID  %CPU %MEM      VSZ    RSS   TT  STAT STARTED      TIME COMMAND
pokotyamu  4991   0.0  0.2  2432876  27172 s008  S+    2:34PM   0:00.38 python use-requests-module.py
USER        PID  %CPU %MEM      VSZ    RSS   TT  STAT STARTED      TIME COMMAND
pokotyamu  4991   0.0  0.2  2432876  27172 s008  S+    2:34PM   0:00.38 python use-requests-module.py
USER        PID  %CPU %MEM      VSZ    RSS   TT  STAT STARTED      TIME COMMAND
pokotyamu  4991   0.0  0.2  2432876  27172 s008  S+    2:34PM   0:00.39 python use-requests-module.py

イテレーター使って回してもそんなに変わらないかな

$ python use-iter.py
USER        PID  %CPU %MEM      VSZ    RSS   TT  STAT STARTED      TIME COMMAND
pokotyamu  8238  48.9  0.2  2433900  27116 s008  S+    2:57PM   0:00.31 python use-iter.py
USER        PID  %CPU %MEM      VSZ    RSS   TT  STAT STARTED      TIME COMMAND
pokotyamu  8238   1.5  0.2  2433900  27128 s008  S+    2:57PM   0:00.32 python use-iter.py
USER        PID  %CPU %MEM      VSZ    RSS   TT  STAT STARTED      TIME COMMAND
pokotyamu  8238   0.0  0.2  2433900  27132 s008  S+    2:57PM   0:00.32 python use-iter.py
USER        PID  %CPU %MEM      VSZ    RSS   TT  STAT STARTED      TIME COMMAND
pokotyamu  8238   0.0  0.2  2433900  27132 s008  S+    2:57PM   0:00.32 python use-iter.py
USER        PID  %CPU %MEM      VSZ    RSS   TT  STAT STARTED      TIME COMMAND
pokotyamu  8238   0.0  0.2  2433900  27136 s008  S+    2:57PM   0:00.32 python use-iter.py
USER        PID  %CPU %MEM      VSZ    RSS   TT  STAT STARTED      TIME COMMAND
pokotyamu  8238   0.0  0.2  2433900  27136 s008  S+    2:57PM   0:00.33 python use-iter.py
USER        PID  %CPU %MEM      VSZ    RSS   TT  STAT STARTED      TIME COMMAND
pokotyamu  8238   0.0  0.2  2433900  27136 s008  S+    2:57PM   0:00.33 python use-iter.py
USER        PID  %CPU %MEM      VSZ    RSS   TT  STAT STARTED      TIME COMMAND
pokotyamu  8238   0.0  0.2  2433900  27136 s008  S+    2:57PM   0:00.33 python use-iter.py
USER        PID  %CPU %MEM      VSZ    RSS   TT  STAT STARTED      TIME COMMAND
pokotyamu  8238   0.0  0.2  2433900  27136 s008  S+    2:57PM   0:00.34 python use-iter.py
USER        PID  %CPU %MEM      VSZ    RSS   TT  STAT STARTED      TIME COMMAND
pokotyamu  8238   0.0  0.2  2433900  27136 s008  S+    2:57PM   0:00.34 python use-iter.py