samuell

import luigi
import os
#from WorkflowUtils import WorkflowUtils
import logging as log 

class ATask(luigi.Task):
    # Task parameters
    dataset_name = luigi.Parameter()
    upstream_task = luigi.Parameter()

samuell

# Some needed luigi imports
import luigi
import luigi.scheduler
import luigi.worker

# Here we are importing our own tasks, provided they are 
# arranged in a python module (folder) named "components"
from components.SomeTaskA import SomeTaskA
from components.SomeTaskB import SomeTaskB
from components.SomeTaskC import SomeTaskC

samuell

'''Luigi Workflow

Usage:
    run_luigi_workflow.py (--host=|--local-scheduler) --workers=<workers> run <taskname> -p <partitioning_parameter>
    run_luigi_workflow.py (-h|--help)
    run_luigi_workflow.py --version

Options:
    -h, --help                      Show this screen
    --version                       Show version

samuell

import luigi
from SomePossibleUpstreamTask import SomePossibleUpstreamTask
from SomeOtherPossibleUpstreamTask import SomeOtherPossibleUpstreamTask

class ExampleTask(luigi.Task):
    upstream_task = luigi.Parameter()
    
    def requires(self):
        # This will return the upstream task object
        return self.upstream_task

samuell

python run_luigi_workflow.py --workers run some_task_c --partitioning-parameter Foo

samuell

[wiki]
user = MyWikiBotUser
pass = ********

samuell

package main

import (
	"bufio"
	"fmt"
	"log"
	"os"
)

func main() {

samuell

class FooParamsMixin(object):
  param1 = luigi.Parameter()
  param2 = luigi.Parameter()
  ...
  
  def foo_params(self):
    return { 'param1': self.param1, 'param2' : self.param2, ... }

class TaskA(FooParamsMixin, luigi.Task):
   def requires(self):

samuell

[formatters]
keys: default

[handlers]
keys: console, logfile

[loggers]
keys: root, luigi-interface

[formatter_default]

samuell

class TaskA1(luigi.Task):
    def requires(self):
        ....
    def output(self):
        return { 'output1' : luigi.LocalTarget( self.input().path + ".output1" ),
                   'output2' : luigi.LocalTarget( self.input().path + ".output2" ) }
    def run(self):
        ....

class TaskA2(luigi.Task):