Perl script for removing document with missing German umlauts from WMT19 Rapid corpus, expects tsv on stdin, produces tsv on stdout.

fixRapid.pl

use strict;
use utf8;

binmode(STDIN, ":utf8");
binmode(STDOUT, ":utf8");

sub checkAndPrint {
  my @doc = @_;
  my @temp = @doc;
  shift(@temp); shift(@temp); shift(@temp); shift(@temp); shift(@temp); # remove first 5 lines, seems to be boilerplate and title sections with umlauts
  my $all = join("\n", @temp); 
  if($all =~ /[äöüß]/i and $all !~ /\b(fr|prsident\p{L}*|europisch\p{L}*)\b/i) {
    print join("\n", @doc), "\n";
  }
}

my @doc;
while(<STDIN>) {
  chomp;
  push(@doc, $_);
  if(/^\s+$/) { # empty line marks document end
    checkAndPrint(@doc);
    @doc = ();
  }
}
checkAndPrint(@doc);