Output Amex JP + Rakuten Card statements as TSV

amextsv.pl

#!/usr/bin/env perl -w

use strict;
use warnings;
use HTTP::Cookies;
use HTTP::Request::Common;
use LWP::UserAgent;
use Web::Scraper;
use Config::Pit;
use Data::Table;

my $config = pit_get("americanexpress.com");

die "You must specify username and password\n    ppit set americanexpress.com"
 unless $config->{password} && $config->{username};

$ENV{PERL_LWP_SSL_VERIFY_HOSTNAME} = 0;

my $ua = LWP::UserAgent->new;
my $cookie_jar = HTTP::Cookies->new;
$ua->cookie_jar($cookie_jar);
$Web::Scraper::UserAgent = $ua;
my ($req, $res);

$req = POST 'https://global.americanexpress.com/myca/logon/japa/action?request_type=LogLogonHandler', 
            [
              'request_type' => 'LogLogonHandler',
              'DestPage' => 'https://global.americanexpress.com/myca/intl/acctsumm/japa/accountSummary.do?request_type=&Face=ja_JP&page=CM',
              'Face' => 'ja_JP',
              'Logon' => 'Continue...',
              'PWD' => '',
              'brandname' => '',
              'TARGET' => '',
              'USERID' => '',
              'CHECKBOXSTATUS' => '',
              'devicePrint' => 'version%3D1%26pm%5Ffpua%3Dmozilla%2F5%2E0%20%28macintosh%3B%20intel%20mac%20os%20x%2010%5F7%5F1%29%20applewebkit%2F534%2E48%2E3%20%28khtml%2C%20like%20gecko%29%20version%2F5%2E1%20safari%2F534%2E48%2E3%7C5%2E0%20%28Macintosh%3B%20Intel%20Mac%20OS%20X%2010%5F7%5F1%29%20AppleWebKit%2F534%2E48%2E3%20%28KHTML%2C%20like%20Gecko%29%20Version%2F5%2E1%20Safari%2F534%2E48%2E3%7CMacIntel%26pm%5Ffpsc%3D24%7C1920%7C1200%7C1140%26pm%5Ffpsw%3D%26pm%5Ffptz%3D9%26pm%5Ffpln%3Dlang%3Den%2Dus%7Csyslang%3D%7Cuserlang%3D%26pm%5Ffpjv%3D1%26pm%5Ffpco%3D1',
              'UserID' => $config->{username},
              'Password' => $config->{password},
            ];

$res = $ua->request($req);

my $txn_scraper = scraper {
  process '//tr[starts-with(@id, "Roc")]', 'txns[]' => scraper {
    process '//td[1]',  date => 'TEXT',
    process '//td[2]',  desc => 'TEXT',
    process '//td[3]',  in   => 'TEXT',
    process '//td[4]',  out  => 'TEXT',
  }
};

my $index = @ARGV ? $ARGV[0] || 1 : 1;
my $ret = $txn_scraper->scrape( URI->new("https://global.americanexpress.com/myca/intl/estatement/japa/statement.do?request_type=&Face=ja_JP&BPIndex=$index&sorted_index=1") );

my $data = [];

foreach my $txn (@{ $ret->{txns} }) {
  $txn->{date} =~ s/[^\d\/]//g;
  $txn->{in} =~ s/[^\d]//g;
  $txn->{out} =~ s/[^\d]//g;
  $txn->{desc} =~ s/^\s*(.+)\s*$/$1/g;
  push @$data, [ $txn->{date}, $txn->{desc}, $txn->{out}, $txn->{in} ];
}

my $t = Data::Table->new($data, [qw(date desc out in)], 0);
print $t->tsv;

rakutentsv.pl

#!/usr/bin/env perl -w

use strict;
use warnings;

use Config::Pit;
use Data::Table;
use HTTP::Cookies;
use HTTP::Request::Common;
use LWP::UserAgent;
use Text::CSV::Encoded;

my $config = pit_get("rakuten-card.co.jp");

die "You must specify userid and password\n    ppit set rakuten-card.co.jp"
  unless $config->{password} && $config->{userid};

my $ua = LWP::UserAgent->new;
my $cookie_jar = HTTP::Cookies->new;
$ua->cookie_jar($cookie_jar);
my ($req, $res);

die 'please specify year and month' unless @ARGV;
my $ym = $ARGV[0];

$req = POST 'https://grp01.id.rakuten.co.jp/rms/nid/login',
  [
    u => $config->{userid},
    p => $config->{password},
    service_id => 'f125',
    param => 'login',
  ];

$res = $ua->request($req);

$req = GET $res->header('location');
$res = $ua->request($req);

$req = POST "https://e-navi.rakuten-card.co.jp/specific/show/tab_no/4/setym/${ym}";
$res = $ua->request($req);

$req = GET 'https://e-navi.rakuten-card.co.jp/specific/download/param/csv';
$res = $ua->request($req);

my $csv = Text::CSV::Encoded->new ({ encoding_in  => "shiftjis" });

my $data = [];
foreach( split /[\n\r]/, $res->content ) {
  $csv->parse($_);
  my @fields = $csv->fields;
  for my $i( 0 ... $#fields ) {
    ( $fields[$i] = $fields[$i] || '' ) =~ s/\s*$//;
  }
  my ( $date, $sign, $user, $subject, $type, $price, $com, $sum ) = @fields;
  next unless $date && $date =~ /\d{2}\.\d{2}\.\d{2}$/ && $subject;
  $date =~ s%\.%\/%g;
  push @$data, [ "20$date", $subject, $user||'', $sum ];
}

my $t = Data::Table->new($data, [qw(date desc user out)], 0);
print $t->tsv;