public
Last active

Shell Pipelines for Python

  • Download Gist
pipelines.py
Python
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72
from subprocess import Popen, PIPE
import re
import glob
import shlex
 
class Pipeline(object):
def __init__(self, descriptor=None):
self.steps = []
if descriptor:
for step in descriptor.split("|"):
self.add(step.strip())
 
def __repr__(self):
return " | ".join(self.steps)
 
def add(self, step, pos=None):
if pos:
self.steps.insert(pos, step)
else:
self.steps.append(step)
pos = len(self.steps) - 1
return pos
 
# syntactic sugar
def __or__(self, step):
self.add(step)
return self
 
def remove(self, pos):
del self.steps[pos]
 
def run(self):
procs = {}
procs[0] = Popen(shlex.split(self.steps[0]), stdout=PIPE)
if len(self.steps) > 1:
i = 1
for p in self.steps[1:]:
procs[i] = Popen(shlex.split(p), stdin=procs[i-1].stdout, stdout=PIPE)
procs[i-1].stdout.close()
output = procs[len(procs) - 1].communicate()[0]
return output
 
class PipelineWithSubstitutions(Pipeline):
def __init__(self, descriptor=None, substitutions=None):
Pipeline.__init__(self, descriptor)
self.substitutions = substitutions
 
def add(self, step, pos=None):
for sub in self.substitutions:
step = re.sub(sub, self.substitutions[sub], step)
Pipeline.add(self, step, pos)
 
if __name__ == "__main__":
import os.path
 
p = Pipeline() | "ls"
print p.run()
p = Pipeline() | "ls -s" | "sort -n"
print p.run()
 
a = ["bib.bib", "test.bib"]
a = []
p = PipelineWithSubstitutions(
substitutions={
"%%": "CURRENT_FILE",
"%:r": os.path.splitext("CURRENT_FILE_N")[0],
"PANDOC#BIBS" : " ".join(["--bibliography " + i for i in a])
}
# ) | "pandoc -t json %%" | "filter" | "pandoc PANDOC#BIBS -f json -t -o latex %:r.latex"
) | "pandoc PANDOC#BIBS -t html -Ss -o %:r.html %%"
print p

Please sign in to comment on this gist.

Something went wrong with that request. Please try again.