Skip to content

Instantly share code, notes, and snippets.

@JohnEmhoff
Created January 27, 2020 02:37
import time
import pyorc
fields = ",".join([f"a{y}:string" for y in range(50)])
spec = f"struct<{fields}>"
for iteration in range(1000):
print(iteration)
writers = []
fps = []
# create 100 writers at a time
for n in range(100):
fp = open(f"out-{n}.orc", "wb")
writers.append(pyorc.Writer(fp, spec, stripe_size=1 * 1024 * 1024))
fps.append(fp)
for fp, w in zip(fps, writers):
w.close()
fp.close()
# slow things down, otherwise it runs away very quickly
time.sleep(1)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment