helper script to download multiple files (components) from a weblate project

weblate-download.pl

#!/usr/bin/perl -CSDA
use strict;
use warnings;
use utf8;

BEGIN { $Pod::Usage::Formatter = 'Pod::Text::Termcap'; }

use LWP::UserAgent;
use HTTP::Request;
use JSON;
use File::Basename;
use File::Path qw(make_path);
use Getopt::Long;
use Pod::Usage;

my $target = '/var/tmp/weblate-download';
my $project = "libo_ui-master";
my $prefix = '';
my $language = '';
my $verbose = '';
my $overwrite = '';
my $keyfile;
my $help = '';

GetOptions('target=s'   => \$target,
           'project:s'  => \$project,
           'filter:s'   => \$prefix,
           'language=s' => \$language,
           'verbose'    => \$verbose,
           'overwrite'  => \$overwrite,
           'keyfile:s'  => \$keyfile,
	       'help|?'     => \$help) or pod2usage(2);

pod2usage(1) if $help;
pod2usage("no language specified - aborting") unless $language;
pod2usage("no project specified - aborting") unless $project;
pod2usage("no target specified - aborting") unless $target;
pod2usage("superfluous arguments (@ARGV) - aborting") if @ARGV;

my $apikey = '';
if (defined $keyfile) {
	$keyfile = "$ENV{HOME}/.config/tdf-weblate" if $keyfile eq '';
	open KEYFILE, "<$keyfile" or pod2usage("keyfile ($keyfile) does not exist (or is not readable) - aborting");
	$apikey = <KEYFILE>;
	close KEYFILE;
	chomp($apikey);
	die "contents of $keyfile don't look like a valid api key\n" unless ($apikey =~ /^[0-9a-zA-Z]{40}$/);
}

my $ua = LWP::UserAgent->new(ssl_opts => { verify_hostname => 1 });
$ua->default_header('Authorization' => "Token $apikey") if $apikey;

my $API_ENDPOINT = 'https://weblate.documentfoundation.org/api/';

my $outdir = "$target/$language";
$outdir .= "/helpcontent2/source" if $project =~ /libo_help/;

# iterate over all components in the project, and download if the prefix matches
my $next = $API_ENDPOINT."projects/$project/components/";
while ($next) {
	print "requesting $next\n" if $verbose;
	my $request = HTTP::Request->new(GET => $next, ['Accept' => 'application/json']);
	my $response = $ua->request($request);
	if ($response->is_success){
		my $hresultref = decode_json $response->decoded_content;
		foreach my $component (@{$hresultref->{'results'}}) {
			my $name = $component->{'name'};
			print "Name: $name\n" if $verbose;
			if ($name =~ /^$prefix/) {
				my($filename, $dirname) = fileparse("$outdir/$name.po");
				if (-e "$dirname/$filename") {
					if ($overwrite) {
						print "$dirname/$filename exists already, but overwrite requested\n" if $verbose;
					} else {
						print "$dirname/$filename exists already, no overwrite requested - skipping!\n";
						next;
					}
				}
				make_path("$dirname");
				my $fileurl = "$API_ENDPOINT/translations/$project/$component->{'slug'}/$language/file/";
				my $porequest = $ua->mirror($fileurl, "$dirname/$filename");
				if ($porequest->is_success) {
					print "Successfully downloaded $dirname/$filename\n";
				} elsif ($porequest->is_error) {
					print "Error: downloading $dirname/$filename failed".$response->status_line."\n";
					print $porequest->error_as_HTML if $verbose;
				}
			}
		}
		$next = $hresultref->{'next'};
	} elsif ($response->is_error){
		print "Error while downloading componentlist: ".$response->status_line."\n";
		print $response->error_as_HTML if $verbose;
		print "aborting... if the message is about too many requests, consult --help for keyfile option\n";
		exit 1;
	}
}
__END__

=pod

=head1 NAME

weblate-download.pl - Download po files in bulk from TDF's weblate server

=head1 SYNOPSIS

weblate-download.pl is used to download multiple po files for a project from
TDF's weblate server.
Downloading will replicate the directory structure as used in the repository.
Note that anonymous access is restricted to 100 requests/day.

Examples (use --help to see all options):

=item C<weblate-download.pl -t /tmp/pofiles -p libo_help-master -l fr>

=item C<weblate-download.pl -k -l vec -f dictionaries>

=head1 AUTHOR

Christian Lohmaier <cloph@documentfoundation.org

=head1 LICENSE

CC0, see L<https://creativecommons.org/publicdomain/zero/1.0/>

=head1 OPTIONS

All options names are case-insensitive and can be abbreviated as long as the
prefix is still unique

=item B<--target>=I</path/to/save/to>

the target directory (will be created if it doesn't exist already) to use for
the download

defaults to /var/tmp/weblate-download

=item B<--project>=I<project>

the project to process - either C<libo_ui-master> or C<libo_help-master>

defaults to libo_ui-master

=item B<--language>=I<lang>

the language to download, required.

=item B<--filter>=I<componenteprefix>

optional option to restrict the download to only those components that start
with C<componentprefix>

=item B<--verbose>

optional switch to output more progress information

=item B<--keyfile>[=I<fileWithApiKey>]

optional switch to enable authenticated API access. If C<--keyfile> is
specified without a pathname, it defaults to F<~/.config/tdf-weblate>

If the option is not specified, anonymous access is used (restricted to 100
requests/day)
The file should contain just your personal API key (found in weblate →
profile-icon → settings → API access)

=item B<--overwrite>

optional switch to overwrite already existing files. If not specified, download
will be skipped for existing files.

Use at your own risk.

=item B<--help>

show this help

=cut

# vim: sw=4 ts=4