Fetching flight info

readme.md

INCOMPLETE - FOR EDUCATIONAL PURPOSES ONLY

I wrote a long twitter thread about developing this Node app, and I thought I'd share the incomplete results

Thread https://twitter.com/TheMapSmith/status/973269523867623425

You definitely shouldn't expect this to work for you!

flights.js

var fs = require('fs');
var request = require('request-promise');
var moment = require('moment')

// Globals
global.timestamp = moment().unix()
global.allPlaybacks = [];
global.geojson = {};
global.geojson['type'] = 'FeatureCollection';
global.geojson['features'] = [];
global.code = "DFW"
global.folder = code + "/"
global.types = ["arrivals", "departures"]
global.FlightIds = [] // array for all IDs found

// first call
makeFolder(getFlights);

function makeFolder(callback) {
  fs.stat(code, function(err, stats) {
    if (err) {
      // Directory doesn't exist or something.
      console.log('Folder doesn\'t exist, so I made the folder ' + code);
      return fs.mkdir(folder, callback);
    }
    if (!stats.isDirectory()) {
      // This isn't a directory!
      callback(new Error('temp is not a directory!'));
    } else {
      console.log('Folder ' + code + ' exists');
      getFlights(code, types);
    }
  });
}

function getFlights() {
  // API URL string slugs
  var uriSource = [
    'https://api.flightradar24.com/common/v1/airport.json?code=',
    '&plugin[]=&plugin-setting[schedule][mode]=',
    '&plugin-setting[schedule][timestamp]=',
    '&page=-1&limit=100&token='
  ]

  // check if we have a file
  types.forEach(function(type) {

    var airportFile = folder + code + "_" + type + ".json"

    // if we don't have an airport file yet
    if (!fs.existsSync(airportFile)) {
      // build the request URL
      var uri = uriSource[0] + code + uriSource[1] + type + uriSource[2] + timestamp + uriSource[3];
      request({
          "method": "GET",
          "uri": uri,
          "headers": {
            "User-Agent": "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/604.5.6 (KHTML, like Gecko) Version/11.0.3 Safari/604.5.6",
            "origin": "https://www.flightradar24.com",
            "referer": "https://www.flightradar24.com/data/flights/ORD"
          }
        })
        .then(function write(response, error) {
          if (!error && response.statusCode == 200) {
            var filename = code + "_" + type + ".json";
            fs.writeFile(folder + filename, response, 'utf8', getIds)
            console.log("Successfully fetched " + filename);
            // TODO: Actually move on with the program from here
          }
        })
        .catch(function(err) {
          console.log("Couldn't fetch " + airportFile + ",code: " + err.statusCode)
        })
    } else if (type === "departures") {
      // only call next function when done with the types array
      getIds(airportFile, types, FlightIds, allPlaybacks);
    } else if (type === "arrivals") {
      console.log("arrivals")
    }
  })
}

// read the Airport file to extract all flightIds
function getIds(airportFile) {
  types.forEach(function(type) {
    var airportFile = folder + code + "_" + type + ".json"
    fs.readFile(airportFile, 'utf8', function(err, data) {
      if (err) throw err;
      var obj = JSON.parse(data);

      var schedule = obj.result.response.airport.pluginData.schedule
      var data = schedule[type].data;

      data.forEach(function(e) {
        var id = e.flight.identification.id
        if (id) {
          FlightIds.push(id)
          console.log(id);
        }
      })
      console.log(type + " ids: " + FlightIds.length);
      if (type === "departures") {
        var i
        processFlightIds(i, FlightIds);
      }
    })
  })
};

function processFlightIds(i) {
  console.log("breakpoint");
  for (var i = FlightIds.length - 1; i >= 0; i--) {
    // drop FlightIds[i] so that i have a call stack based on each flight id
    fetchPlayback(FlightIds[i], i);
  }
  if (FlightIds.length === 0) {
    processTracks(writeGeoJSON)
  }
};

// fetch the playback files for each flightId
function fetchPlayback(flightId, i) {
  var flightObj = {}
  console.log("getting " + flightId);
  var uriSource = [
    'https://api.flightradar24.com/common/v1/flight-playback.json?flightId=',
    '&timestamp=',
    '&token='
  ]
  var filename = folder + "flight-playback-" + flightId + ".json";

  if (!fs.existsSync(filename)) {
    var uri = uriSource[0] + flightId + uriSource[1] + timestamp + uriSource[2]
    // TODO: Catch request errors
    // TODO: Add delay tool - try npm bottleneck https://stackoverflow.com/questions/43715068/how-to-use-the-bottleneck-npm-module

    request({
        "method": "GET",
        "uri": uri,
        "headers": {
          "User-Agent": "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/604.5.6 (KHTML, like Gecko) Version/11.0.3 Safari/604.5.6",
          "origin": "https://www.flightradar24.com",
          "referer": "https://www.flightradar24.com/data/flights/ORD"
        }
      })
      .then(function write(response) {
        if (!error && response.statusCode == 200) {
          fs.writeFile(filename, response, 'utf8')
          // make an array object out of the current flight info and push into a
          // master array that will be processed all at once
          flightObj.file = filename
          flightObj.id = flightId
          allPlaybacks.push(flightObj)
          // drop a FlightId from the array to whittle down the call stack
          FlightIds.pop() // drop the final one
        }
      })
      .catch(function(err) {
        console.log("Couldn't fetch " + filename + ",code: " + err.statusCode)
      })

  } else {
    // make an array object out of the current flight info and push into a
    // master array that will be processed all at once
    flightObj.file = filename
    flightObj.id = flightId
    allPlaybacks.push(flightObj)
    // drop a FlightId from the array to whittle down the call stack
    FlightIds.pop() // drop the final one
  }
}


// take a Flight Playback json and turn the coordinates into a GeoJSON line
function processTracks(callback) {
  // read all the files
  console.log("huh")
  for (var i = 0; i < allPlaybacks.length; i++) {
    if (i === allPlaybacks.length) {
      callback(geojson)
    }
    var onePlayback = allPlaybacks[i];
    fs.readFile(onePlayback.file, 'utf8', function(err, data) {
      if (err) throw err;
      var obj = JSON.parse(data);
      // var flightID = obj.result.request.flightId
      var track = obj.result.response.data.flight.track;

      var coordinates = []

      if (track) {
        track.forEach(function(e) {
          var pair = []
          pair.push(e.longitude)
          pair.push(e.latitude)
          coordinates.push(pair)
        })

        var feature = {
          "type": "Feature",
          "geometry": {
            "type": "LineString",
            "coordinates": coordinates
          },
          "properties": {}
        }
        geojson['features'].push(feature);

      }
    })
  }
};

function writeGeoJSON() {
  var filename = code + '.geoJSON'
  fs.writeFile(filename, JSON.stringify(geojson), 'utf8');
}

Hey @TheMapSmith

Looks like you got there, well done! Couple of minor comments to try and help

• Make use of qs option when setting up your request.
  • that'll save you from having to do things like var uri = uriSource[0] + flightId + uriSource[1] + timestamp + uriSource[2]
• Another great option for making requests is a library called axios
  • It let's you set up a base instance of a request and then modify little bits for certain requests, this would save you a bit of duplicated config.
  • Another neat thing from axios which may have helped you is how you can send a few requests off simultantiously and then wait for all their resolutions, see the bottom of this example entitled 'Performing multiple concurrent requests'

Anyway good job, well done!

Can we have the package.json file 😃 ?

Hi Luke, thanks for taking an interest! How did you come across this?

I'd have to dig around to find the source on my old laptop, but I think all that would be in the package.json are the libraries that you see referenced at the top in the require() section.

Thank you. I think I just started googling flight path data api, and eventually got to an article you wrote 😄

I did try to get it to work with fetch, and so I think I have an example that is correct in terms of structure, but the api call was failing I will come back to it.

I joined a small artistic community a while back, one of the guys there was retrieving flight data from an api. I've always loved flight paths so tried to see if it might be possible to find these and then render them with svg or similar.

Check Langlands and Bell as an artist couple who made some really nice stuff with flight paths (if you don't know them already).

Nice, thanks for sharing the artistic inspiration!

http://www.langlandsandbell.com/portfolio-item/air-routes-of-britain-night-day-2000/

The API call was failing

I eventually stopped trying to use FlightRadar24 because the code here was hitting an unpublished API, and my IP eventually go blacklisted by their CDN. Maybe that's the issue you were hitting?