Skip to content

Instantly share code, notes, and snippets.

@LorePep
Last active November 26, 2018 07:46
Show Gist options
  • Save LorePep/9785f01f3b755a51891b64f986d12989 to your computer and use it in GitHub Desktop.
Save LorePep/9785f01f3b755a51891b64f986d12989 to your computer and use it in GitHub Desktop.
Simple pipeline in luigi
import luigi
class WritePipelineTask(luigi.Task):
def output(self):
return luigi.LocalTarget("data/output_one.txt")
def run(self):
with self.output().open("w") as output_file:
output_file.write("pipeline")
class AddMyTask(luigi.Task):
def output(self):
return luigi.LocalTarget("data/output_two.txt")
def requires(self):
return WritePipelineTask()
def run(self):
with self.input().open("r") as input_file:
line = input_file.read()
with self.output().open("w") as output_file:
decorated_line = "My "+line
output_file.write(decorated_line)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment