Last active
December 22, 2015 21:17
-
-
Save belden/db642bcb97d1f49cec39 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
use strict; | |
use warnings; | |
package Functional::ForkManager; | |
use base qw(Exporter); | |
use Scalar::Util qw(blessed); | |
use Parallel::ForkManager; | |
use Functional::Iterator; | |
our @EXPORT = qw(fork_over these_things); | |
our $MAX_KIDS = 5; | |
sub these_things (&;@) { @_ } | |
sub fork_over(&;@) { | |
my $child_code = shift; | |
my ($work_generator, @whoops_catchall) = shift->(); | |
$work_generator = blessed($work_generator) && $work_generator->can('next') | |
? $work_generator | |
: iterator(records => [$work_generator, @whoops_catchall]); | |
my %args = (-max_kids => $MAX_KIDS, @_); | |
my $pm = Parallel::ForkManager->new($args{-max_kids}); | |
LOOP: while (my $work = $work_generator->next) { | |
$pm->start and next LOOP; | |
$child_code->($work); | |
$pm->finish; | |
} | |
$pm->wait_all_children; | |
} | |
1; | |
__END__ | |
=pod | |
=head1 NAME | |
Functional::ForkManager - some kind of misguided wrapper on top of Parallel::ForkManager | |
=head1 SYNOPSIS | |
Run some code across some list. If `these_things` yields an object that can `->next`, then that | |
object will be treated as an iterator/generator to produce work arguments for the forked kids. | |
Simple usage with `these_things` just returning a list of things to work on: | |
=over 4 | |
use Functional::ForkManager; | |
$FunctionalForkManager::MAX_KIDS = 10; # package-level configuration | |
fork_over { print "kid $$ got: $_[0]\n" } | |
these_things { 1..10 } | |
-max_kids => 23; | |
=back | |
A more dynamic example where `these_things` fetches work as needed. This might | |
do something like spin up 17 kids in parallel and generate 100-ish random | |
numbers. I haven't run any of this so do not know for sure! | |
=over 4 | |
use Functional::ForkManager; | |
use Functional::Iterator qw(iterator); | |
fork_over { print "kid $$ got: $_[0]\n" } | |
these_things { | |
my $max_numbers = 100; | |
iterator(generator => sub { | |
return undef if --$max_numbers <= 0; | |
return int(rand($max_numbers)); | |
}); | |
} | |
-max_kids => 17; | |
=back | |
And this might iterate the records from your database handle and do something with them. | |
=over 4 | |
use DBI; | |
use Functional::ForkManager; | |
use Functional::Iterator qw(iterator); | |
my $dbh = DBI->connect('somehow'); | |
fork_over { | |
my $row = shift; | |
Some::Class->work_on($row); | |
} | |
these_things { | |
my $sth = $dbh->prepare('select foo from bar where id > 1'); | |
$sth->execute; | |
return iterator(generator => sub { | |
my $next_rec = $sth->fetchrow_hashref; | |
$sth->finish if ! defined $next_rec; | |
return $next_rec; | |
}); | |
} | |
-max_kids => 10; | |
=bacK | |
=head1 WHY | |
WHY WHY WHY | |
=head1 DISCLAIMER | |
Experimental untested code is experimental and untested | |
=head1 AUTHOR | |
@belden | |
=cut |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment