Compare Webmaker Discourse users to Webmaker users

process-discourse-users.js

// This script generates a Markdown-formatted report to help with the
// following GitHub issue:
//
//   https://github.com/mozilla/teach.webmaker.org/issues/930
//
// Before running it, install the dependencies:
//
//   npm install underscore superagent event-stream progress

var fs = require('fs');
var _ = require('underscore');
var request = require('superagent');
var es = require('event-stream');
var ProgressBar = require('progress');

// This should be in the form of 'user:pass'.
var LOGINAPI_AUTH = process.env.LOGINAPI_AUTH;

// This should point to a CSV export of the Discourse users, which
// can be done via the Discourse admin panel.
var CSV_FILE = 'discourse-users.csv';

var LOGINAPI_ORIGIN = 'https://login.webmaker.org';
var DISCOURSE_ORIGIN = 'https://discourse.webmaker.org';
var CACHE_FILE = '.process-discourse-users.cache.json';

var cache;

var users = fs.readFileSync(CSV_FILE, 'utf-8')
  .trim().split('\n').map(function(line, i) {
    var columns = line.trim().split(',');
    if (columns.length != 4) {
      throw new Error('error at line ' + i);
    }
    return {
      id: parseInt(columns[0].trim()),
      name: columns[1].trim(),
      username: columns[2].trim(),
      email: columns[3].trim()
    };
  });

var setCacheEntry = function(key, value) {
  cache[key] = value;
  fs.writeFileSync(CACHE_FILE, JSON.stringify(cache, null, 2));
};

if (!fs.existsSync(CACHE_FILE)) {
  fs.writeFileSync(CACHE_FILE, '{}');
}

cache = JSON.parse(fs.readFileSync(CACHE_FILE, 'utf-8'));

function getUsers() {
  return es.readArray(users)
    .pipe(es.through(function(user) {
      var done = function() {
        this.resume();
        this.emit('data', _.extend({
          webmakerUsername: cache[user.username]
        }, user));
      }.bind(this);
      this.pause();
      if (user.username in cache) {
        process.nextTick(done);
      } else {
        request
          .get(LOGINAPI_ORIGIN + '/user/email/' + user.email)
          .auth(LOGINAPI_AUTH.split(':')[0], LOGINAPI_AUTH.split(':')[1])
          .end(function(err, res) {
            if (err) {
              if (err.status === 404) {
                setCacheEntry(user.username, null);
                return done();
              }
              return this.emit('error', err);
            }
            setCacheEntry(user.username, res.body.user.username);
            done();
          }.bind(this));
      }
    }));
}

function getDiscourseStats(users) {
  return es.readArray(users)
    .pipe(es.through(function(user) {
      var key = '__discourse_info_' + user.username;
      var done = function() {
        this.resume();
        this.emit('data', _.extend({
          hasPosted: !!cache[key].user.last_posted_at
        }, user));
      }.bind(this);
      this.pause();
      if (key in cache) {
        process.nextTick(done);
      } else {
        request
          .get(DISCOURSE_ORIGIN + '/users/' + user.username + '.json')
          .end(function(err, res) {
            if (err) {
              return this.emit('error', err);
            }
            setCacheEntry(key, res.body);
            done();
          }.bind(this));
      }
    }));
}

function main() {
  var hasIdenticalWebmakerAccount = [];
  var hasNoWebmakerAccount = [];
  var needsUsernameChange = [];
  var nameConflicts = [];
  var hasPosted = [];
  var userLinks = {};
  var linkTo = function(username) {
    userLinks[username] = DISCOURSE_ORIGIN + '/users/' + username;
    return '[' + username + ']';
  };
  var bar = new ProgressBar('retrieving webmaker data [:bar] :percent', {
    total: users.length
  });

  getUsers().on('data', function(user) {
    bar.tick();
    if (user.webmakerUsername) {
      if (user.webmakerUsername === user.username) {
        hasIdenticalWebmakerAccount.push(user);
      } else {
        if (_.findWhere(users, { username: user.webmakerUsername })) {
          nameConflicts.push(user);
        } else {
          needsUsernameChange.push(user);
        }
      }
    } else {
      hasNoWebmakerAccount.push(user);
    }
  }).on('end', function() {
    console.log();
    bar = new ProgressBar('retrieving discourse data [:bar] :percent', {
      total: hasNoWebmakerAccount.length
    });
    getDiscourseStats(hasNoWebmakerAccount).on('data', function(user) {
      bar.tick();
      if (user.hasPosted) {
        hasPosted.push(user);
      }
    }).on('end', function() {
      console.log('1. Discourse users with identical Webmaker usernames: ' +
                  hasIdenticalWebmakerAccount.length);
      console.log('2. Discourse users with reconcilable Webmaker usernames: ' +
                  needsUsernameChange.length);
      console.log('3. Discourse users with conflicting Webmaker usernames: ' +
                  nameConflicts.length);
      nameConflicts.forEach(function(user) {
        console.log("  * " + linkTo(user.username) + ' / ' +
                    linkTo(user.webmakerUsername));
      });
      console.log('4. Discourse users without a known webmaker account: ' +
                  hasNoWebmakerAccount.length);
      console.log("  * Who have posted at least once: " +
                  hasPosted.length);
      hasPosted.forEach(function(user) {
        console.log("    * " + linkTo(user.username));
      });
      console.log();

      Object.keys(userLinks).forEach(function(username) {
        console.log('[' + username + ']: ' + userLinks[username]);
      });
    });
  });
}

if (!module.parent) {
  main();
}