Sketch for processing a queue of files (several instances can be launched in parallel)

process_file_queue.pl

use strict;
use warnings;
use File::Spec;

my $in = 'in';
my $out = 'out';

sub process {
  my ($in, $out, $callback) = @_;
  opendir (my $DIR, $in) or die "Failed to open dir $in: $!";
  for my $file (readdir $DIR) {
    next if $file=~/^\./;
    my $in_file = File::Spec->catfile($in, $file);
    my $out_file = File::Spec->catfile($out, $file);
    if (rename($in_file, $out_file)) {
      $callback->($out_file);
    }
  }
}

sub write_to_s3 {
  my ($xml) = @_;
  print "Writing $xml to s3: ";
  open (my $IN, '<', $xml) or die "Coudln't open $xml: $!";
  print do {local $/; <$IN>};
  print "\n";
  unlink($xml);
}

while (1) {
  process($in, $out, \&write_to_s3);
}