Skip to content

Instantly share code, notes, and snippets.

View samuell's full-sized avatar
Hacking away

Samuel Lampa samuell

Hacking away
View GitHub Profile
import luigi
import os
#from WorkflowUtils import WorkflowUtils
import logging as log
class ATask(luigi.Task):
# Task parameters
dataset_name = luigi.Parameter()
upstream_task = luigi.Parameter()
# Some needed luigi imports
import luigi
import luigi.scheduler
import luigi.worker
# Here we are importing our own tasks, provided they are
# arranged in a python module (folder) named "components"
from components.SomeTaskA import SomeTaskA
from components.SomeTaskB import SomeTaskB
from components.SomeTaskC import SomeTaskC
'''Luigi Workflow
Usage: (--host=|--local-scheduler) --workers=<workers> run <taskname> -p <partitioning_parameter> (-h|--help) --version
-h, --help Show this screen
--version Show version
import luigi
from SomePossibleUpstreamTask import SomePossibleUpstreamTask
from SomeOtherPossibleUpstreamTask import SomeOtherPossibleUpstreamTask
class ExampleTask(luigi.Task):
upstream_task = luigi.Parameter()
def requires(self):
# This will return the upstream task object
return self.upstream_task
python --workers run some_task_c --partitioning-parameter Foo
samuell / myconfig.cfg
Last active August 29, 2015 14:01
Query Semantic MediaWiki via MediaWiki REST API
user = MyWikiBotUser
pass = ********
package main
import (
func main() {
samuell /
Created June 4, 2014 16:11
Snippet showing how to define (and pass on) workflow-wide parameters in the luigi workflow system. Courtesy of Joe Ennever ("stolen" from!msg/luigi-user/7zIS_qseNOM/TWcYtRe5SVoJ )
class FooParamsMixin(object):
param1 = luigi.Parameter()
param2 = luigi.Parameter()
def foo_params(self):
return { 'param1': self.param1, 'param2' : self.param2, ... }
class TaskA(FooParamsMixin, luigi.Task):
def requires(self):
keys: default
keys: console, logfile
keys: root, luigi-interface
class TaskA1(luigi.Task):
def requires(self):
def output(self):
return { 'output1' : luigi.LocalTarget( self.input().path + ".output1" ),
'output2' : luigi.LocalTarget( self.input().path + ".output2" ) }
def run(self):
class TaskA2(luigi.Task):