Skip to content

Instantly share code, notes, and snippets.

@jlinoff
Last active February 1, 2022 23:47
Show Gist options
  • Save jlinoff/0f7b290dc4e1f58ad803 to your computer and use it in GitHub Desktop.
Save jlinoff/0f7b290dc4e1f58ad803 to your computer and use it in GitHub Desktop.
Python example that shows how to copy a large file showing progress.
#!/usr/bin/env python
'''
Copy a large file showing progress.
MIT License
Copyright (c) 2015 Joe Linoff
Permission is hereby granted, free of charge, to any person
obtaining a copy of this software and associated documentation
files (the "Software"), to deal in the Software without
restriction, including without limitation the rights to use,
copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following
conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
OTHER DEALINGS IN THE SOFTWARE.
'''
import math
import os
import sys
import time
def copy_large_file(src, dst):
'''
Copy a large file showing progress.
'''
print('copying "{}" --> "{}"'.format(src, dst))
if os.path.exists(src) is False:
print('ERROR: file does not exist: "{}"'.format(src))
sys.exit(1)
if os.path.exists(dst) is True:
os.remove(dst)
if os.path.exists(dst) is True:
print('ERROR: file exists, cannot overwrite it: "{}"'.format(dst))
sys.exit(1)
# Start the timer and get the size.
start = time.time()
size = os.stat(src).st_size
print('{} bytes'.format(size))
# Adjust the chunk size to the input size.
divisor = 10000 # .1%
#chunk_size = size / divisor
chunk_size = math.ceil( size / divisor ) # suggested by 0xmessi to fix an error.
while chunk_size == 0 and divisor > 0:
divisor /= 10
chunk_size = size / divisor
print('chunk size is {}'.format(chunk_size))
# Copy.
try:
with open(src, 'rb') as ifp:
with open(dst, 'wb') as ofp:
copied = 0 # bytes
chunk = ifp.read(chunk_size)
while chunk:
# Write and calculate how much has been written so far.
ofp.write(chunk)
copied += len(chunk)
per = 100. * float(copied) / float(size)
# Calculate the estimated time remaining.
elapsed = time.time() - start # elapsed so far
avg_time_per_byte = elapsed / float(copied)
remaining = size - copied
est = remaining * avg_time_per_byte
est1 = size * avg_time_per_byte
eststr = 'rem={:>.1f}s, tot={:>.1f}s'.format(est, est1)
# Write out the status.
sys.stdout.write('\r\033[K{:>6.1f}% {} {} --> {} '.format(per, eststr, src, dst))
sys.stdout.flush()
# Read in the next chunk.
chunk = ifp.read(chunk_size)
except IOError as obj:
print('\nERROR: {}'.format(obj))
sys.exit(1)
sys.stdout.write('\r\033[K') # clear to EOL
elapsed = time.time() - start
print('copied "{}" --> "{}" in {:>.1f}s"'.format(src, dst, elapsed))
if __name__ == '__main__':
copy_large_file(sys.argv[1], sys.argv[2])
@0xmessi
Copy link

0xmessi commented Nov 15, 2021

hey its giving an error actually

copying "ErrorReport.html" --> "./output/ff.html"
14640 bytes
chunk size is 1.464
Traceback (most recent call last):
  File "/Users/mrana/Documents/testing/checkIt.py", line 94, in <module>
    copy_large_file(sys.argv[1], sys.argv[2])
  File "/Users/mrana/Documents/testing/checkIt.py", line 62, in copy_large_file
    chunk = ifp.read(chunk_size)
TypeError: argument should be integer or None, not 'float'

@0xmessi
Copy link

0xmessi commented Nov 15, 2021

use chunk = math.ceil( size / divisor ) on line 56 instead

@jlinoff
Copy link
Author

jlinoff commented Nov 15, 2021

Thank you @0xmessi , i really appreciated you tracking down and fixing the problem.

@0xmessi
Copy link

0xmessi commented Nov 16, 2021

Also, you need to import math

@jlinoff
Copy link
Author

jlinoff commented Nov 21, 2021

thank you again

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment