Last active
April 7, 2021 09:18
-
-
Save MartinEnders/e95a11ca66943c97a177f555e611d39a to your computer and use it in GitHub Desktop.
Read multiple files line by line in Python evaluated as lazy as possible
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import itertools | |
import glob | |
print(list(itertools.chain([1,2,3],[4,5,6],[7,'a']))) | |
print("-"*80) | |
### LAZY | |
# Writing files for test | |
for i in range(1,4): | |
with open(f'tmp/test{i}.txt','w') as out: | |
for x in range(1,10): | |
print(f"File: {out.name}, Line: {x} {str(x*i)*x*i}",file=out) | |
out.write("---\n") | |
# round parennthesis: generator | |
# iglob: iterable | |
file_handles = (open(x) for x in glob.iglob("tmp/*.txt")) | |
# iterate over list of filehandles: | |
for line in itertools.chain.from_iterable(file_handles): | |
print(f"Length of line: {len(line):>4} | {line}",end='') |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
[1, 2, 3, 4, 5, 6, 7, 'a'] | |
-------------------------------------------------------------------------------- | |
Length of line: 31 | File: tmp/test1.txt, Line: 1 1 | |
Length of line: 32 | File: tmp/test1.txt, Line: 2 22 | |
Length of line: 33 | File: tmp/test1.txt, Line: 3 333 | |
Length of line: 34 | File: tmp/test1.txt, Line: 4 4444 | |
Length of line: 35 | File: tmp/test1.txt, Line: 5 55555 | |
Length of line: 36 | File: tmp/test1.txt, Line: 6 666666 | |
Length of line: 37 | File: tmp/test1.txt, Line: 7 7777777 | |
Length of line: 38 | File: tmp/test1.txt, Line: 8 88888888 | |
Length of line: 39 | File: tmp/test1.txt, Line: 9 999999999 | |
Length of line: 4 | --- | |
Length of line: 32 | File: tmp/test2.txt, Line: 1 22 | |
Length of line: 34 | File: tmp/test2.txt, Line: 2 4444 | |
Length of line: 36 | File: tmp/test2.txt, Line: 3 666666 | |
Length of line: 38 | File: tmp/test2.txt, Line: 4 88888888 | |
Length of line: 50 | File: tmp/test2.txt, Line: 5 10101010101010101010 | |
Length of line: 54 | File: tmp/test2.txt, Line: 6 121212121212121212121212 | |
Length of line: 58 | File: tmp/test2.txt, Line: 7 1414141414141414141414141414 | |
Length of line: 62 | File: tmp/test2.txt, Line: 8 16161616161616161616161616161616 | |
Length of line: 66 | File: tmp/test2.txt, Line: 9 181818181818181818181818181818181818 | |
Length of line: 4 | --- | |
Length of line: 33 | File: tmp/test3.txt, Line: 1 333 | |
Length of line: 36 | File: tmp/test3.txt, Line: 2 666666 | |
Length of line: 39 | File: tmp/test3.txt, Line: 3 999999999 | |
Length of line: 54 | File: tmp/test3.txt, Line: 4 121212121212121212121212 | |
Length of line: 60 | File: tmp/test3.txt, Line: 5 151515151515151515151515151515 | |
Length of line: 66 | File: tmp/test3.txt, Line: 6 181818181818181818181818181818181818 | |
Length of line: 72 | File: tmp/test3.txt, Line: 7 212121212121212121212121212121212121212121 | |
Length of line: 78 | File: tmp/test3.txt, Line: 8 242424242424242424242424242424242424242424242424 | |
Length of line: 84 | File: tmp/test3.txt, Line: 9 272727272727272727272727272727272727272727272727272727 | |
Length of line: 4 | --- |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment