Skip to content

Instantly share code, notes, and snippets.

@dimaspivak
Created June 14, 2019 18:24
Show Gist options
  • Save dimaspivak/697832905390ee61ab2d7a85cbe1e77e to your computer and use it in GitHub Desktop.
Save dimaspivak/697832905390ee61ab2d7a85cbe1e77e to your computer and use it in GitHub Desktop.
from streamsets.sdk import DataCollector
SDC_URL = 'localhost:18630'
SDC_USERNAME = 'admin'
SDC_PASSWORD = 'admin'
STAGE_NAME = 'com_streamsets_pipeline_stage_destination_mapreduce_MapReduceDExecutor'
LEGACY_LIBRARY = 'streamsets-datacollector-cdh_5_12-lib'
NEW_LIBRARY = 'streamsets-datacollector-cdh_5_15-lib'
data_collector = DataCollector(SDC_URL, SDC_USERNAME, SDC_PASSWORD)
for pipeline in data_collector.pipelines:
contains_stage = False
for stage in pipeline:
if stage.stage_name == STAGE_NAME and stage.library == LEGACY_LIBRARY:
print('Found stage ({}) using legacy library in pipeline (id={}, title={}) ...'.format(stage.instance_name,
pipeline.id,
pipeline.title))
contains_stage = True
stage.library = NEW_LIBRARY
if contains_stage:
print('Updating pipeline ...')
data_collector.remove_pipeline(pipeline)
data_collector.add_pipeline(pipeline)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment