Last active
June 6, 2019 01:31
-
-
Save phluid61/8817db6a20d217b44cc128ef41e5bd42 to your computer and use it in GitHub Desktop.
Run generate_views in parallel: `MAX_PARALLEL_CHILDREN=4 bin/generate_views_parallel myrepo`
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/perl -w | |
use FindBin; | |
use lib "$FindBin::Bin/../perl_lib"; | |
use EPrints; | |
use strict; | |
use Getopt::Long; | |
use Pod::Usage; | |
use POSIX qw(strftime); | |
use Carp; | |
my $DEBUG = 1; | |
#local $SIG{__WARN__} = sub { confess @_; }; | |
sub now { | |
return strftime '%F %T', localtime(time); | |
} | |
# $childprocs : HASHREF of { $pid => _, ... } | |
# $limit : int - wait until $childprocs has fewer entries than this | |
sub flush_childprocs { | |
my( $childprocs, $limit ) = @_; | |
$limit //= 1; | |
while( $limit && ( scalar keys %$childprocs >= $limit )) | |
{ | |
my $pid = wait; | |
if( $pid == -1 ) | |
{ | |
print STDERR "fork(): expected ",(scalar keys %$childprocs)," child processes, but wait() returned -1\n"; | |
delete $childprocs->{ keys %$childprocs }; | |
} | |
else | |
{ | |
delete $childprocs->{$pid}; | |
} | |
} | |
} | |
my $verbose = 0; | |
my $quiet = 0; | |
my $help = 0; | |
my $man = 0; | |
my $generate_opt; | |
my $lang_opt; | |
Getopt::Long::Configure("permute"); | |
GetOptions( | |
'help|?' => \$help, | |
'man' => \$man, | |
'verbose+' => \$verbose, | |
'silent' => \$quiet, | |
'quiet' => \$quiet, | |
'generate=s' => \$generate_opt, | |
'lang=s' => \$lang_opt, | |
) || pod2usage(2); | |
pod2usage(1) if $help; | |
pod2usage(-exitstatus => 0, -verbose => 2) if $man; | |
pod2usage(2) if(scalar @ARGV != 1); | |
my $noise = 1; | |
$noise = 0 if($quiet); | |
$noise = 1+$verbose if($verbose); | |
my $do_menus = 1; | |
my $do_lists = 1; | |
if (defined $generate_opt) { | |
if ($generate_opt ne 'menus' and $generate_opt ne 'lists') { | |
print STDERR "--generate must be either menus or lists.\n"; | |
exit 1; | |
} | |
} | |
$|=1; | |
binmode(STDOUT, ':utf8'); | |
my $repoid = $ARGV[0]; | |
if (!defined $repoid) { | |
print STDERR "Usage: $0 <repoid>\n"; | |
exit 1; | |
} | |
my $session = new EPrints::Session(1 , $repoid, $noise); | |
if (!defined $session) { | |
print STDERR "Failed to load repository: $repoid\n"; | |
exit 1; | |
} | |
my $repository = $session->get_repository; | |
my $views = $repository->get_conf('browse_views'); | |
my $MAX_PARALLEL_CHILDREN = 1; | |
if( defined $ENV{MAX_PARALLEL_CHILDREN} && $ENV{MAX_PARALLEL_CHILDREN} >= 0 ) | |
{ | |
$MAX_PARALLEL_CHILDREN = 0 + $ENV{MAX_PARALLEL_CHILDREN}; | |
} | |
my %childprocs = (); | |
VIEW: foreach my $view (@{$views}) { | |
my $pid = fork(); | |
if( $pid ) | |
{ | |
$childprocs{$pid} = 1; | |
# if there are enough child processes running, wait for some to | |
# finish before spawning more | |
flush_childprocs( \%childprocs, $MAX_PARALLEL_CHILDREN ); | |
} | |
else | |
{ | |
print '[', now, '] beginning ', $view->{id}, "\n" if $DEBUG || $noise > 1; | |
my @args = ('/opt/eprints3/bin/generate_views', $repoid, '--view', $view->{id}); | |
if ($generate_opt) { | |
push @args, '--generate', $generate_opt; | |
} | |
if ($noise < 1) { | |
push @args, '--silent'; | |
} else { | |
my $v = $verbose; | |
while ($v) { | |
push @args, '--verbose'; | |
$v --; | |
} | |
} | |
if ($lang_opt) { | |
push @args, '--lang', $lang_opt; | |
} | |
print '[', now, '] > ', (join ' ', @args), "\n" if $DEBUG || $noise > 2; | |
system @args; | |
print '[', now, '] finished ', $view->{id}, "\n" if $DEBUG || $noise > 1; | |
exit; | |
} | |
} | |
# wait for any remaining child processes to terminate | |
flush_childprocs( \%childprocs ); | |
$session->terminate(); | |
exit; | |
=pod | |
=for Pod2Wiki | |
=head1 NAME | |
B<generate_views_parallel> - Generate static browse pages for an EPrint repository | |
=head1 SYNOPSIS | |
B<generate_views_parallel> I<repository_id> [B<options>] | |
=head1 DESCRIPTION | |
This script invokes the standard EPrints B<generate_views> script, but does so once per view, to work around a possible memory leak in that script. | |
B<generate_views> creates some or all of the pages used in the /view/ section of the website. Since 3.1 these pages update themselves if they get only than a certain age, but this can cause delays for the viewer, so this script is still provided to pre-prepare them to provide a smoother experience. | |
Note: Since EPrints 3.1 it is not essential to run generate_views periodically, but it is still recommended. | |
=head1 ARGUMENTS | |
=over 8 | |
=item B<repository_id> | |
The ID of the eprint repository to use. | |
=back | |
=head1 OPTIONS | |
=over 8 | |
=item B<--generate menus> | |
Only generate the menu pages, not the pages with links to records. | |
=item B<--generate lists> | |
Only generate the pages with lists of records, not the menu pages. | |
=item B<--lang> I<lang_id> | |
Generate only pages for the language with this ID. | |
=item B<--help> | |
Print a brief help message and exit. | |
=item B<--man> | |
Print the full manual page and then exit. | |
=item B<--quiet> | |
Be vewwy vewwy quiet. This option will suppress all output unless an error occurs. | |
=item B<--verbose> | |
Explain in detail what is going on. | |
May be repeated for greater effect. | |
=back | |
=head1 COPYRIGHT | |
=for COPYRIGHT BEGIN | |
Copyright 2019 Queensland University of Technology. | |
=for COPYRIGHT END | |
=for LICENSE BEGIN | |
This file is part of EPrints L<http://www.eprints.org/>. | |
EPrints is free software: you can redistribute it and/or modify it | |
under the terms of the GNU General Public License as published | |
by the Free Software Foundation, either version 3 of the License, or | |
(at your option) any later version. | |
EPrints is distributed in the hope that it will be useful, but WITHOUT | |
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public | |
License for more details. | |
You should have received a copy of the GNU General Public License | |
along with EPrints. If not, see L<http://www.gnu.org/licenses/>. | |
=for LICENSE END | |
=cut |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment