Skip to content

Instantly share code, notes, and snippets.

@sambvfx sambvfx/external.py
Created Nov 27, 2019

Embed
What would you like to do?
Beam bug when using multiple ExternalTransforms.
from __future__ import print_function
import apache_beam as beam
from apache_beam.io.external.generate_sequence import GenerateSequence
def main(options):
pipe = beam.Pipeline(options=options)
p1 = (
pipe
| 'P1' >> GenerateSequence(start=1, stop=5, expansion_service='localhost:8097')
)
p2 = (
pipe
| 'P2' >> GenerateSequence(start=5, stop=10, expansion_service='localhost:8097')
)
(p1, p2) | beam.Flatten() | beam.Map(print)
# print(pipe.to_runner_api())
result = pipe.run()
result.wait_until_finish()
if __name__ == '__main__':
import argparse
from apache_beam.options.pipeline_options import PipelineOptions
parser = argparse.ArgumentParser()
_, pipeline_args = parser.parse_known_args()
main(PipelineOptions(pipeline_args))
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.