Recursively crawl outgoing transactions from an origin bitcoin address

btcrawl.js

const fs = require('fs');
const request = require('request');
const rp = require('request-promise');
const sb = require('satoshi-bitcoin');
const d3 = Object.assign({},require('d3-array'));

// the address to start crawling outgoing transactions from
const START_ADDRESS = '1Ftixp78FjTWFi3ssJjBw5NqKf5ZPQjXBb';
// the starting timestamp to compare transactions against
const START_DATE = new Date(1499413652000);
// collect transactions within this number of hours from start 
const HOUR_LIMIT = 8;
// millisecond delay between hits to the Blockchain.info API
// keep this number high, do not abuse the API
const RATE_LIMIT_MS = 2000;

// before crawling, be sure to create an empty tx.json in same folder
let transactions = JSON.parse(fs.readFileSync(__dirname+"/tx.json", 'utf8'));
let usedOrigins = [];
let usedAddrs = [];
let step = 0;

const getTransactions = (address) => {
  // do not crawl from same origin twice
  if (usedOrigins.indexOf(address) === -1) {
    usedOrigins.push(address);
    step++;
  } else {
    return;
  }
  let newTransactions = [];
  rp({url:"https://blockchain.info/rawaddr/"+address, json: true})
    .then(body => {
      // loop through all of this wallet's transactions
      for (let tx of body.txs) {
        // check if this is a withdraw by looking at inputs
        const OUTGOING_SATOSHI = d3.sum(tx.inputs,(input) => {
          // if the input address matches this wallet, add it to sum
          if (input.prev_out.addr == address) {
            return +input.prev_out.value;
          }
        });
        // check if transaction occurred within x hours
        const HOURS = Math.abs(new Date(tx.time*1000) - START_DATE) / 36e5;
        // does it meet both requirements
        if (OUTGOING_SATOSHI > 0 && HOURS >=0 && HOURS <=HOUR_LIMIT && tx.out.length > 0) {
          for (let out of tx.out){
            // is a spent output
            if (out.spent == true) {
              // check if this output address has shown up before
              // if so, mark it has multiple parents and give unique id
              let outAddress = out.addr+"-"+step;
              let hasMultiple = true;
              if (usedAddrs.indexOf(out.addr) === -1) {
                usedAddrs.push(out.addr);
                outAddress = out.addr;
                hasMultiple = false;
              }
              newTransactions.push({
                out_addr: out.addr,
                address: outAddress,
                parent_address: address,
                value: out.value,
                value_btc: sb.toBitcoin(out.value),
                tx_hash: tx.hash,
                unix_time: tx.time,
                num_parent_txs: body.txs.length,
                has_multiple_parents: hasMultiple
              });
            }
          }
        }
      }
      console.log("returning "+newTransactions.length);
      return (newTransactions.length > 0) ? newTransactions : null;
    })
    .then(data => {
      // add new data to saved transactions and write to file
      if (data) {
        transactions = transactions.concat(data);
        fs.writeFileSync(__dirname+"/tx.json", JSON.stringify(transactions), 'utf8');
        return data;
      }
    })
    .then(data => {
      // queue newly obtained outgoing addresses for crawling
      if (data) {
        let i = 0;
        let intId = setInterval(() => {
          if (data[i].out_addr) getTransactions(data[i].out_addr);
          i++;
          if (i >= data.length) clearInterval(intId);
        },RATE_LIMIT_MS);
      }
    })
    .catch(err => {
      console.log("ERROR FROM "+address);
      console.log("ERROR CODE "+err);
    });
}

getTransactions(START_ADDRESS);