Created
January 21, 2018 13:12
-
-
Save xianminx/fc528fbf9a64dd88852d1ce701587403 to your computer and use it in GitHub Desktop.
weird clocking in python datetime.now()
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from concurrent.futures import ProcessPoolExecutor | |
import random,os | |
from datetime import datetime | |
import multiprocessing | |
from multiprocessing import Process, Queue, Manager | |
from multiprocessing.sharedctypes import Value, Array | |
import time, random | |
import threading | |
m = multiprocessing.Manager() | |
queue = m.Queue() | |
def worker(queue, index, logfile): | |
starttime = datetime.now() | |
print('%s: %s/%s:start worker: index=%s' % (str(starttime),os.getpid(), threading.currentThread().getName(), index)) | |
line = 'radfjljfads fljk asdjfajskdfjklas dflkasdjflkjvlckxzjlkjasdflkjasdlkfasdf' | |
lines = [line for i in xrange(5000000)] | |
convertline = '\n'.join(lines) | |
filename = '/tmp/aaa%s.%s' % (int(time.time()*1000), random.randint(10000, 99999)) | |
with open(filename, 'w+') as f: | |
f.write(convertline) | |
print('%s/%s: %s----%s' % (os.getpid(), threading.currentThread().getName(), filename, len(convertline))) | |
returntime = datetime.now() | |
queue.put((index, starttime, returntime, logfile, filename)) | |
# return (index, starttime, returntime, logfile, filename) | |
def done_callback(): | |
done_callback_starttime = datetime.now() | |
try: | |
print('done_callback') | |
(index, starttime, returntime, logfile, filename) = queue.get(block=True) | |
with open(filename, 'r') as f: | |
convertline = f.read() | |
print('%s++++++%s' % (filename, len(convertline))) | |
os.remove(filename) | |
print('done: %s' % len(convertline)) | |
endtime = datetime.now() | |
diff = endtime-starttime # total time | |
diff2 = returntime - | |
# worker time | |
diff3 = done_callback_starttime - returntime # queue time | |
diff4 = endtime - done_callback_starttime # done time | |
print('%s - %s: pid(%s/%s):done_callback: index=%s, takes(%s, %s, %s, %s)' % (str(endtime), str(starttime), os.getpid(), threading.currentThread().getName(), index, diff, diff2, diff3, diff4)) | |
except Exception as e: | |
print(e) | |
def done_worker(): | |
while True: | |
# print('quesize = %s' % len(queue)) | |
done_callback() | |
def que(): | |
qmax = 1000 | |
t = threading.Thread(target=done_worker) | |
t.name='done_worker' | |
t.start() | |
with ProcessPoolExecutor() as executor: | |
for index in xrange(10000): | |
# print('%s: pid(%s):submit: index=%s' % (str(datetime.now()), os.getpid(), index)) | |
while len(executor._pending_work_items) > qmax: | |
time.sleep(3) | |
future = executor.submit(worker, queue, index, 'filename_'+str(index)) | |
# future.add_done_callback(done_callback) | |
def main(): | |
que() | |
time.sleep(100) | |
if __name__ == '__main__': | |
main() |
Author
xianminx
commented
Jan 21, 2018
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment