Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
helper script to download multiple files (components) from a weblate project
#!/usr/bin/perl -CSDA
use strict;
use warnings;
use utf8;
BEGIN { $Pod::Usage::Formatter = 'Pod::Text::Termcap'; }
use LWP::UserAgent;
use HTTP::Request;
use JSON;
use File::Basename;
use File::Path qw(make_path);
use Getopt::Long;
use Pod::Usage;
my $target = '/var/tmp/weblate-download';
my $project = "libo_ui-master";
my $prefix = '';
my $language = '';
my $verbose = '';
my $overwrite = '';
my $keyfile;
my $help = '';
GetOptions('target=s' => \$target,
'project:s' => \$project,
'filter:s' => \$prefix,
'language=s' => \$language,
'verbose' => \$verbose,
'overwrite' => \$overwrite,
'keyfile:s' => \$keyfile,
'help|?' => \$help) or pod2usage(2);
pod2usage(1) if $help;
pod2usage("no language specified - aborting") unless $language;
pod2usage("no project specified - aborting") unless $project;
pod2usage("no target specified - aborting") unless $target;
pod2usage("superfluous arguments (@ARGV) - aborting") if @ARGV;
my $apikey = '';
if (defined $keyfile) {
$keyfile = "$ENV{HOME}/.config/tdf-weblate" if $keyfile eq '';
open KEYFILE, "<$keyfile" or pod2usage("keyfile ($keyfile) does not exist (or is not readable) - aborting");
$apikey = <KEYFILE>;
close KEYFILE;
chomp($apikey);
die "contents of $keyfile don't look like a valid api key\n" unless ($apikey =~ /^[0-9a-zA-Z]{40}$/);
}
my $ua = LWP::UserAgent->new(ssl_opts => { verify_hostname => 1 });
$ua->default_header('Authorization' => "Token $apikey") if $apikey;
my $API_ENDPOINT = 'https://weblate.documentfoundation.org/api/';
my $outdir = "$target/$language";
$outdir .= "/helpcontent2/source" if $project =~ /libo_help/;
# iterate over all components in the project, and download if the prefix matches
my $next = $API_ENDPOINT."projects/$project/components/";
while ($next) {
print "requesting $next\n" if $verbose;
my $request = HTTP::Request->new(GET => $next, ['Accept' => 'application/json']);
my $response = $ua->request($request);
if ($response->is_success){
my $hresultref = decode_json $response->decoded_content;
foreach my $component (@{$hresultref->{'results'}}) {
my $name = $component->{'name'};
print "Name: $name\n" if $verbose;
if ($name =~ /^$prefix/) {
my($filename, $dirname) = fileparse("$outdir/$name.po");
if (-e "$dirname/$filename") {
if ($overwrite) {
print "$dirname/$filename exists already, but overwrite requested\n" if $verbose;
} else {
print "$dirname/$filename exists already, no overwrite requested - skipping!\n";
next;
}
}
make_path("$dirname");
my $fileurl = "$API_ENDPOINT/translations/$project/$component->{'slug'}/$language/file/";
my $porequest = $ua->mirror($fileurl, "$dirname/$filename");
if ($porequest->is_success) {
print "Successfully downloaded $dirname/$filename\n";
} elsif ($porequest->is_error) {
print "Error: downloading $dirname/$filename failed".$response->status_line."\n";
print $porequest->error_as_HTML if $verbose;
}
}
}
$next = $hresultref->{'next'};
} elsif ($response->is_error){
print "Error while downloading componentlist: ".$response->status_line."\n";
print $response->error_as_HTML if $verbose;
print "aborting... if the message is about too many requests, consult --help for keyfile option\n";
exit 1;
}
}
__END__
=pod
=head1 NAME
weblate-download.pl - Download po files in bulk from TDF's weblate server
=head1 SYNOPSIS
weblate-download.pl is used to download multiple po files for a project from
TDF's weblate server.
Downloading will replicate the directory structure as used in the repository.
Note that anonymous access is restricted to 100 requests/day.
Examples (use --help to see all options):
=item C<weblate-download.pl -t /tmp/pofiles -p libo_help-master -l fr>
=item C<weblate-download.pl -k -l vec -f dictionaries>
=head1 AUTHOR
Christian Lohmaier <cloph@documentfoundation.org
=head1 LICENSE
CC0, see L<https://creativecommons.org/publicdomain/zero/1.0/>
=head1 OPTIONS
All options names are case-insensitive and can be abbreviated as long as the
prefix is still unique
=item B<--target>=I</path/to/save/to>
the target directory (will be created if it doesn't exist already) to use for
the download
defaults to /var/tmp/weblate-download
=item B<--project>=I<project>
the project to process - either C<libo_ui-master> or C<libo_help-master>
defaults to libo_ui-master
=item B<--language>=I<lang>
the language to download, required.
=item B<--filter>=I<componenteprefix>
optional option to restrict the download to only those components that start
with C<componentprefix>
=item B<--verbose>
optional switch to output more progress information
=item B<--keyfile>[=I<fileWithApiKey>]
optional switch to enable authenticated API access. If C<--keyfile> is
specified without a pathname, it defaults to F<~/.config/tdf-weblate>
If the option is not specified, anonymous access is used (restricted to 100
requests/day)
The file should contain just your personal API key (found in weblate →
profile-icon → settings → API access)
=item B<--overwrite>
optional switch to overwrite already existing files. If not specified, download
will be skipped for existing files.
Use at your own risk.
=item B<--help>
show this help
=cut
# vim: sw=4 ts=4
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.