Created
November 11, 2015 05:22
-
-
Save LusciousPear/138b20cb36d5b4aa3e69 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
(def test-graph ;each pipeline can only be defined in an origin once? | |
{:provider {:account-file "blah.json" :project "hx-test" :region "us-central1"} | |
:pipelines {"Pipeline1" | |
{:transform-graph ["AbstractTransform/AbstractTransform.jar"]} | |
"Pipeline2" | |
{:transform-graph ["AbstractTransform/AbstractTransform2.jar"]} | |
"Pipeline3" | |
{:transform-graph ["PythonScripts/PipelinePy3.py"]} | |
"Pipeline4" | |
{:transform-graph ["AbstractTransform/Pipeline4.jar"]} | |
"Pipeline5" | |
{:transform-graph ["PythonScripts/Pipeline5.py"]} | |
#_"RepairJob" | |
#_{:transform-graph ["PythonScripts/RepairClickstream.py"]} | |
} | |
:sources [{:package "ClickStream"} {:package "ShoppingCartClickStream"} {:package "CartTransaction"} {:package "EmailClickStream"}] | |
:sinks [{:package "S3Sink"} {:package "EnhancedAdwordsSink"} {:package "EnhancedAdwordsSink"} {:package "CartTrans-Redshift"} {:package "EmailClicks-RedShift"} {:package "RawEmailS3"}] | |
:edges [{:origin "Pipeline1" :targets ["Pipeline2"]} | |
{:origin "ClickStream" :targets ["Pipeline1"]} | |
{:origin "Pipeline2" | |
:error-sink :google-storage ;implied sink of gcs directory "Pipeline2/errors/blah" | |
:error-handler "RepairJob" ;implied source of gcs directory "Pipeline2/errors/blah" | |
:targets ["S3Sink" "EnhancedAdwordsSink"]} | |
{:origin "ShoppingCartClickStream" :targets ["Pipeline3"]} | |
{:origin "Pipeline3" :targets ["Pipeline2"]} | |
{:origin "CartTransaction" :targets ["Pipeline4"]} | |
{:origin "Pipeline4" :targets ["CartTrans-Redshift" ]} ;implied {:name "CartTrans-Redshift"}, and write to a RedShift table and write to the post-processing queue of Pipeline2? {"Pipeline2" :type "post"} | |
{:origin "EmailClickstream" :targets ["Pipeline5" "RawEmailS3"]} | |
{:origin "Pipeline5" :targets ["Emailclicks-RedShift"]}]}) | |
[ { | |
"resource" : { | |
"google_pubsub" : { | |
"EmailClickstream-out" : { | |
"name" : "EmailClickstream-out" | |
} | |
} | |
} | |
}, { | |
"resource" : { | |
"google_pubsub" : { | |
"EmailClickstream-err" : { | |
"name" : "EmailClickstream-err" | |
} | |
} | |
} | |
}, { | |
"resource" : { | |
"google_pubsub" : { | |
"Pipeline5-out" : { | |
"name" : "Pipeline5-out" | |
} | |
} | |
} | |
}, { | |
"resource" : { | |
"google_pubsub" : { | |
"Pipeline5-err" : { | |
"name" : "Pipeline5-err" | |
} | |
} | |
} | |
}, { | |
"resource" : { | |
"google_pubsub" : { | |
"Pipeline3-out" : { | |
"name" : "Pipeline3-out" | |
} | |
} | |
} | |
}, { | |
"resource" : { | |
"google_pubsub" : { | |
"Pipeline3-err" : { | |
"name" : "Pipeline3-err" | |
} | |
} | |
} | |
}, { | |
"resource" : { | |
"google_pubsub" : { | |
"Pipeline2-out" : { | |
"name" : "Pipeline2-out" | |
} | |
} | |
} | |
}, { | |
"resource" : { | |
"google_pubsub" : { | |
"Pipeline2-err" : { | |
"name" : "Pipeline2-err" | |
} | |
} | |
} | |
}, { | |
"resource" : { | |
"google_pubsub" : { | |
"CartTransaction-out" : { | |
"name" : "CartTransaction-out" | |
} | |
} | |
} | |
}, { | |
"resource" : { | |
"google_pubsub" : { | |
"CartTransaction-err" : { | |
"name" : "CartTransaction-err" | |
} | |
} | |
} | |
}, { | |
"resource" : { | |
"google_pubsub" : { | |
"Pipeline4-out" : { | |
"name" : "Pipeline4-out" | |
} | |
} | |
} | |
}, { | |
"resource" : { | |
"google_pubsub" : { | |
"Pipeline4-err" : { | |
"name" : "Pipeline4-err" | |
} | |
} | |
} | |
}, { | |
"resource" : { | |
"google_pubsub" : { | |
"Pipeline1-out" : { | |
"name" : "Pipeline1-out" | |
} | |
} | |
} | |
}, { | |
"resource" : { | |
"google_pubsub" : { | |
"Pipeline1-err" : { | |
"name" : "Pipeline1-err" | |
} | |
} | |
} | |
}, { | |
"resource" : { | |
"google_pubsub" : { | |
"ShoppingCartClickStream-out" : { | |
"name" : "ShoppingCartClickStream-out" | |
} | |
} | |
} | |
}, { | |
"resource" : { | |
"google_pubsub" : { | |
"ShoppingCartClickStream-err" : { | |
"name" : "ShoppingCartClickStream-err" | |
} | |
} | |
} | |
}, { | |
"resource" : { | |
"google_pubsub" : { | |
"ClickStream-out" : { | |
"name" : "ClickStream-out" | |
} | |
} | |
} | |
}, { | |
"resource" : { | |
"google_pubsub" : { | |
"ClickStream-err" : { | |
"name" : "ClickStream-err" | |
} | |
} | |
} | |
}, { | |
"resource" : { | |
"google_pubsub" : { | |
"RawEmailS3-error" : { | |
"name" : "RawEmailS3-error" | |
} | |
} | |
} | |
}, { | |
"resource" : { | |
"google_pubsub" : { | |
"Emailclicks-RedShift-error" : { | |
"name" : "Emailclicks-RedShift-error" | |
} | |
} | |
} | |
}, { | |
"resource" : { | |
"google_pubsub" : { | |
"S3Sink-error" : { | |
"name" : "S3Sink-error" | |
} | |
} | |
} | |
}, { | |
"resource" : { | |
"google_pubsub" : { | |
"EnhancedAdwordsSink-error" : { | |
"name" : "EnhancedAdwordsSink-error" | |
} | |
} | |
} | |
}, { | |
"resource" : { | |
"google_pubsub" : { | |
"CartTrans-Redshift-error" : { | |
"name" : "CartTrans-Redshift-error" | |
} | |
} | |
} | |
}, { | |
"resource" : { | |
"google_dataflow" : { | |
"EmailClickstream" : { | |
"name" : "EmailClickstream", | |
"class" : "Main.class", | |
"depends_on" : [ "google_pubsub.Pipeline5-out", "google_pubsub.RawEmailS3-out" ], | |
"project" : "hx-test" | |
} | |
} | |
} | |
}, { | |
"resource" : { | |
"google_dataflow" : { | |
"Pipeline5" : { | |
"name" : "Pipeline5", | |
"class" : "Main.class", | |
"depends_on" : [ "google_pubsub.Emailclicks-RedShift-out", "google_pubsub.EmailClickstream-out", "google_dataflow.EmailClickstream" ], | |
"project" : "hx-test" | |
} | |
} | |
} | |
}, { | |
"resource" : { | |
"google_dataflow" : { | |
"RawEmailS3" : { | |
"name" : "RawEmailS3", | |
"class" : "Main.class", | |
"depends_on" : [ "google_pubsub.EmailClickstream-out", "google_dataflow.EmailClickstream" ], | |
"project" : "hx-test" | |
} | |
} | |
} | |
}, { | |
"resource" : { | |
"google_dataflow" : { | |
"Emailclicks-RedShift" : { | |
"name" : "Emailclicks-RedShift", | |
"class" : "Main.class", | |
"depends_on" : [ "google_pubsub.Pipeline5-out", "google_dataflow.Pipeline5" ], | |
"project" : "hx-test" | |
} | |
} | |
} | |
}, { | |
"resource" : { | |
"google_dataflow" : { | |
"Pipeline3" : { | |
"name" : "Pipeline3", | |
"class" : "Main.class", | |
"depends_on" : [ "google_pubsub.Pipeline2-out", "google_pubsub.ShoppingCartClickStream-out", "google_dataflow.ShoppingCartClickStream" ], | |
"project" : "hx-test" | |
} | |
} | |
} | |
}, { | |
"resource" : { | |
"google_dataflow" : { | |
"Pipeline2" : { | |
"name" : "Pipeline2", | |
"class" : "Main.class", | |
"depends_on" : [ "google_pubsub.S3Sink-out", "google_pubsub.EnhancedAdwordsSink-out", "google_pubsub.Pipeline3-out", "google_pubsub.Pipeline1-out", "google_dataflow.Pipeline3", "google_dataflow.Pipeline1" ], | |
"project" : "hx-test" | |
} | |
} | |
} | |
}, { | |
"resource" : { | |
"google_dataflow" : { | |
"S3Sink" : { | |
"name" : "S3Sink", | |
"class" : "Main.class", | |
"depends_on" : [ "google_pubsub.Pipeline2-out", "google_dataflow.Pipeline2" ], | |
"project" : "hx-test" | |
} | |
} | |
} | |
}, { | |
"resource" : { | |
"google_dataflow" : { | |
"EnhancedAdwordsSink" : { | |
"name" : "EnhancedAdwordsSink", | |
"class" : "Main.class", | |
"depends_on" : [ "google_pubsub.Pipeline2-out", "google_dataflow.Pipeline2" ], | |
"project" : "hx-test" | |
} | |
} | |
} | |
}, { | |
"resource" : { | |
"google_dataflow" : { | |
"CartTransaction" : { | |
"name" : "CartTransaction", | |
"class" : "Main.class", | |
"depends_on" : [ "google_pubsub.Pipeline4-out" ], | |
"project" : "hx-test" | |
} | |
} | |
} | |
}, { | |
"resource" : { | |
"google_dataflow" : { | |
"Pipeline4" : { | |
"name" : "Pipeline4", | |
"class" : "Main.class", | |
"depends_on" : [ "google_pubsub.CartTrans-Redshift-out", "google_pubsub.CartTransaction-out", "google_dataflow.CartTransaction" ], | |
"project" : "hx-test" | |
} | |
} | |
} | |
}, { | |
"resource" : { | |
"google_dataflow" : { | |
"CartTrans-Redshift" : { | |
"name" : "CartTrans-Redshift", | |
"class" : "Main.class", | |
"depends_on" : [ "google_pubsub.Pipeline4-out", "google_dataflow.Pipeline4" ], | |
"project" : "hx-test" | |
} | |
} | |
} | |
}, { | |
"resource" : { | |
"google_dataflow" : { | |
"Pipeline1" : { | |
"name" : "Pipeline1", | |
"class" : "Main.class", | |
"depends_on" : [ "google_pubsub.Pipeline2-out", "google_pubsub.ClickStream-out", "google_dataflow.ClickStream" ], | |
"project" : "hx-test" | |
} | |
} | |
} | |
}, { | |
"resource" : { | |
"google_dataflow" : { | |
"ShoppingCartClickStream" : { | |
"name" : "ShoppingCartClickStream", | |
"class" : "Main.class", | |
"depends_on" : [ "google_pubsub.Pipeline3-out" ], | |
"project" : "hx-test" | |
} | |
} | |
} | |
}, { | |
"resource" : { | |
"google_dataflow" : { | |
"ClickStream" : { | |
"name" : "ClickStream", | |
"class" : "Main.class", | |
"depends_on" : [ "google_pubsub.Pipeline1-out" ], | |
"project" : "hx-test" | |
} | |
} | |
} | |
} ] |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment