Fetching flight info

readme.md

INCOMPLETE - FOR EDUCATIONAL PURPOSES ONLY

I wrote a long twitter thread about developing this Node app, and I thought I'd share the incomplete results

Thread https://twitter.com/TheMapSmith/status/973269523867623425

You definitely shouldn't expect this to work for you!

flights.js

var fs = require('fs');
var request = require('request-promise');
var moment = require('moment')

// Globals
global.timestamp = moment().unix()
global.allPlaybacks = [];
global.geojson = {};
global.geojson['type'] = 'FeatureCollection';
global.geojson['features'] = [];
global.code = "DFW"
global.folder = code + "/"
global.types = ["arrivals", "departures"]
global.FlightIds = [] // array for all IDs found

// first call
makeFolder(getFlights);

function makeFolder(callback) {
  fs.stat(code, function(err, stats) {
    if (err) {
      // Directory doesn't exist or something.
      console.log('Folder doesn\'t exist, so I made the folder ' + code);
      return fs.mkdir(folder, callback);
    }
    if (!stats.isDirectory()) {
      // This isn't a directory!
      callback(new Error('temp is not a directory!'));
    } else {
      console.log('Folder ' + code + ' exists');
      getFlights(code, types);
    }
  });
}

function getFlights() {
  // API URL string slugs
  var uriSource = [
    'https://api.flightradar24.com/common/v1/airport.json?code=',
    '&plugin[]=&plugin-setting[schedule][mode]=',
    '&plugin-setting[schedule][timestamp]=',
    '&page=-1&limit=100&token='
  ]

  // check if we have a file
  types.forEach(function(type) {

    var airportFile = folder + code + "_" + type + ".json"

    // if we don't have an airport file yet
    if (!fs.existsSync(airportFile)) {
      // build the request URL
      var uri = uriSource[0] + code + uriSource[1] + type + uriSource[2] + timestamp + uriSource[3];
      request({
          "method": "GET",
          "uri": uri,
          "headers": {
            "User-Agent": "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/604.5.6 (KHTML, like Gecko) Version/11.0.3 Safari/604.5.6",
            "origin": "https://www.flightradar24.com",
            "referer": "https://www.flightradar24.com/data/flights/ORD"
          }
        })
        .then(function write(response, error) {
          if (!error && response.statusCode == 200) {
            var filename = code + "_" + type + ".json";
            fs.writeFile(folder + filename, response, 'utf8', getIds)
            console.log("Successfully fetched " + filename);
            // TODO: Actually move on with the program from here
          }
        })
        .catch(function(err) {
          console.log("Couldn't fetch " + airportFile + ",code: " + err.statusCode)
        })
    } else if (type === "departures") {
      // only call next function when done with the types array
      getIds(airportFile, types, FlightIds, allPlaybacks);
    } else if (type === "arrivals") {
      console.log("arrivals")
    }
  })
}

// read the Airport file to extract all flightIds
function getIds(airportFile) {
  types.forEach(function(type) {
    var airportFile = folder + code + "_" + type + ".json"
    fs.readFile(airportFile, 'utf8', function(err, data) {
      if (err) throw err;
      var obj = JSON.parse(data);

      var schedule = obj.result.response.airport.pluginData.schedule
      var data = schedule[type].data;

      data.forEach(function(e) {
        var id = e.flight.identification.id
        if (id) {
          FlightIds.push(id)
          console.log(id);
        }
      })
      console.log(type + " ids: " + FlightIds.length);
      if (type === "departures") {
        var i
        processFlightIds(i, FlightIds);
      }
    })
  })
};

function processFlightIds(i) {
  console.log("breakpoint");
  for (var i = FlightIds.length - 1; i >= 0; i--) {
    // drop FlightIds[i] so that i have a call stack based on each flight id
    fetchPlayback(FlightIds[i], i);
  }
  if (FlightIds.length === 0) {
    processTracks(writeGeoJSON)
  }
};

// fetch the playback files for each flightId
function fetchPlayback(flightId, i) {
  var flightObj = {}
  console.log("getting " + flightId);
  var uriSource = [
    'https://api.flightradar24.com/common/v1/flight-playback.json?flightId=',
    '&timestamp=',
    '&token='
  ]
  var filename = folder + "flight-playback-" + flightId + ".json";

  if (!fs.existsSync(filename)) {
    var uri = uriSource[0] + flightId + uriSource[1] + timestamp + uriSource[2]
    // TODO: Catch request errors
    // TODO: Add delay tool - try npm bottleneck https://stackoverflow.com/questions/43715068/how-to-use-the-bottleneck-npm-module

    request({
        "method": "GET",
        "uri": uri,
        "headers": {
          "User-Agent": "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/604.5.6 (KHTML, like Gecko) Version/11.0.3 Safari/604.5.6",
          "origin": "https://www.flightradar24.com",
          "referer": "https://www.flightradar24.com/data/flights/ORD"
        }
      })
      .then(function write(response) {
        if (!error && response.statusCode == 200) {
          fs.writeFile(filename, response, 'utf8')
          // make an array object out of the current flight info and push into a
          // master array that will be processed all at once
          flightObj.file = filename
          flightObj.id = flightId
          allPlaybacks.push(flightObj)
          // drop a FlightId from the array to whittle down the call stack
          FlightIds.pop() // drop the final one
        }
      })
      .catch(function(err) {
        console.log("Couldn't fetch " + filename + ",code: " + err.statusCode)
      })

  } else {
    // make an array object out of the current flight info and push into a
    // master array that will be processed all at once
    flightObj.file = filename
    flightObj.id = flightId
    allPlaybacks.push(flightObj)
    // drop a FlightId from the array to whittle down the call stack
    FlightIds.pop() // drop the final one
  }
}


// take a Flight Playback json and turn the coordinates into a GeoJSON line
function processTracks(callback) {
  // read all the files
  console.log("huh")
  for (var i = 0; i < allPlaybacks.length; i++) {
    if (i === allPlaybacks.length) {
      callback(geojson)
    }
    var onePlayback = allPlaybacks[i];
    fs.readFile(onePlayback.file, 'utf8', function(err, data) {
      if (err) throw err;
      var obj = JSON.parse(data);
      // var flightID = obj.result.request.flightId
      var track = obj.result.response.data.flight.track;

      var coordinates = []

      if (track) {
        track.forEach(function(e) {
          var pair = []
          pair.push(e.longitude)
          pair.push(e.latitude)
          coordinates.push(pair)
        })

        var feature = {
          "type": "Feature",
          "geometry": {
            "type": "LineString",
            "coordinates": coordinates
          },
          "properties": {}
        }
        geojson['features'].push(feature);

      }
    })
  }
};

function writeGeoJSON() {
  var filename = code + '.geoJSON'
  fs.writeFile(filename, JSON.stringify(geojson), 'utf8');
}

Nice, thanks for sharing the artistic inspiration!

http://www.langlandsandbell.com/portfolio-item/air-routes-of-britain-night-day-2000/

The API call was failing

I eventually stopped trying to use FlightRadar24 because the code here was hitting an unpublished API, and my IP eventually go blacklisted by their CDN. Maybe that's the issue you were hitting?