Skip to content

Instantly share code, notes, and snippets.

@allenday

allenday/Makefile

Last active Aug 6, 2020
Embed
What would you like to do?
coingecko-etl
#!/usr/bin/perl
use strict;
use JSON;
my $JJ = new JSON;
$JJ->canonical(1);
my $id = shift @ARGV;
open(X, qq(curl --silent -X GET "https://api.coingecko.com/api/v3/coins/$id/ohlc?vs_currency=usd&days=max" -H "accept: application/json" |));
my $ohlc = $JJ->decode( join '', <X> );
close(X);
foreach my $t ( @$ohlc ) {
$t->[0] /= 1000;
print join(",", ($id, @$t)), "\n";
}
sleep 1.0;
#!/usr/bin/perl
use strict;
use utf8;
use Encode;
use Data::Dumper qw(Dumper);
use JSON;
binmode(STDOUT, ':encoding(utf8)');
my $JJ = new JSON;
$JJ->canonical(1);
my $id = shift @ARGV;
open(X, qq(curl --silent -X GET "https://api.coingecko.com/api/v3/coins/$id" -H "accept: application/json" |));
my $meta = $JJ->decode( join '', map {decode('UTF-8',$_)} <X> );
close(X);
my $symbol = $meta->{symbol};
my $twitter = $meta->{links}->{twitter_screen_name};
my $logo = $meta->{image}->{large};
$symbol = undef if $symbol eq "";
$twitter = undef if $twitter eq "";
$logo = undef if $logo eq "";
my $contract_address = $meta->{contract_address};
print $JJ->encode({id => lc($id), symbol => lc($symbol), twitter => lc($twitter), contract_address => lc($contract_address), logo => $logo});
print "\n";
#print STDERR Dumper($meta);
sleep 1.0;
#!/usr/bin/perl
use strict;
use Data::Dumper;
use JSON;
my $JJ = new JSON;
$JJ->canonical(1);
open(X, 'curl --silent -X GET "https://api.coingecko.com/api/v3/coins/list" -H "accept: application/json" |');
my $coins = $JJ->decode( join '', <X> );
close(X);
foreach my $coin ( @$coins ) {
my $id = $coin->{id};
print qq(make $id.ohlc.csv;\n);
print qq(make $id.meta.jsonl;\n);
}
%.ohlc.csv :
perl ./coingecko-csv.pl ${@:.ohlc.csv=} > $@
%.meta.jsonl :
perl ./coingecko-metadata.pl ${@:.meta.jsonl=} > $@
all.meta.jsonl :
cat `ls *.meta.jsonl | grep -v $@` | perl -ne 'print lc' | sort > $@
all.ohlc.csv :
cat `ls *.ohlc.csv | grep -v $@` > $@
all :
perl ./coingecko.pl | bash
$(MAKE) all.meta.jsonl
$(MAKE) all.ohlc.csv
gsutil cp all.meta.jsonl gs://crypto-public-data/
gsutil cp all.ohlc.csv gs://crypto-public-data/
clean :
rm *.meta.jsonl
rm *.ohlc.csv
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.