Skip to content

Instantly share code, notes, and snippets.

@sambvfx
Created November 27, 2019 01:16
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save sambvfx/e8b08acac8ded19e86ecefaa79a8a272 to your computer and use it in GitHub Desktop.
Save sambvfx/e8b08acac8ded19e86ecefaa79a8a272 to your computer and use it in GitHub Desktop.
Beam bug when using multiple ExternalTransforms.
from __future__ import print_function
import apache_beam as beam
from apache_beam.io.external.generate_sequence import GenerateSequence
def main(options):
pipe = beam.Pipeline(options=options)
p1 = (
pipe
| 'P1' >> GenerateSequence(start=1, stop=5, expansion_service='localhost:8097')
)
p2 = (
pipe
| 'P2' >> GenerateSequence(start=5, stop=10, expansion_service='localhost:8097')
)
(p1, p2) | beam.Flatten() | beam.Map(print)
# print(pipe.to_runner_api())
result = pipe.run()
result.wait_until_finish()
if __name__ == '__main__':
import argparse
from apache_beam.options.pipeline_options import PipelineOptions
parser = argparse.ArgumentParser()
_, pipeline_args = parser.parse_known_args()
main(PipelineOptions(pipeline_args))
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment