Skip to content

Instantly share code, notes, and snippets.

@l1n
Created November 5, 2020 02:33
Show Gist options
  • Save l1n/69e391ed95708a2b683b3e00c81ffd3b to your computer and use it in GitHub Desktop.
Save l1n/69e391ed95708a2b683b3e00c81ffd3b to your computer and use it in GitHub Desktop.
#!/bin/perl
use strict;
use warnings;
use utf8;
use JSON;
use HTML::WikiConverter;
my $DEBUG = 0;
my $wc = new HTML::WikiConverter( dialect => 'Markdown', encoding => 'utf8', link_style => 'inline' );
my ($start_date, $end_date);
$start_date = `date +%s`;
chomp $start_date;
$start_date = 1604528758 if $DEBUG;
my %ids;
while (1) {
$end_date = `date +%s`;
chomp $end_date;
print "$end_date\n" if $DEBUG;
my $current_entries = `curl https://fivethirtyeight.com/wp-json/liveblog/v1/291003/entries/$start_date/$end_date/ -s`;
$current_entries =~ s/[^\x00-\x7f]//g;
$current_entries = decode_json $current_entries;
if ($#{$current_entries->{entries}} >= 0) {
foreach my $e (@{$current_entries->{entries}}) {
print "$e->{id}\n";
next if $ids{$e->{id}};
$ids{$e->{id}} = 1;
$e->{content} =~ s/[^\x00-\x7f]//g;
next unless $e->{authors}[0]->{avatar};
my ($avatar) = ($e->{authors}[0]->{avatar} =~ /src="(.+?)"/);
my $content = $wc->html2wiki(html => $e->{content}, base_uri => 'https://fivethirtyeight.com');
$content =~ s/<br \\>//g;
$content =~ s/<table>/```/g;
$content =~ s/<\/table>\n/```/g;
$content =~ s/<\/tr>\n/\n/g;
$content =~ s/<\/td>\n/\t/g;
$content =~ s/<th>\n?/\t/g;
$content =~ s/<t[hrd]>\n?//g;
$content =~ s/\n>\n/\n/g;
foreach (unpack("(A2000)*", $content)) {
my %payload = (
content => $_,
username => "$e->{authors}[0]->{name} [$e->{type}]",
avatar_url => "$avatar"
);
my $payload = encode_json \%payload;
print "$payload\n";
my @webhook_url = @ARGV;
foreach (@webhook_url) {
my @args = ("curl", "-H", 'Content-Type: application/json', $_, '--data', $payload);
system (@args);
sleep 1;
}
}
}
}
$start_date = $end_date;
sleep 5;
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment