Importing users from MediaWiki to FxA

accountscreator.js

/**
 * Import MediaWiki accounts
 *
 * To use within fxa-auth-server deployment.
 *
 * Based on work pushed in https://github.com/mozilla/fxa-auth-server/pull/711
 *
 * See documentation at http://docs.webplatform.org/wiki/WPD:Projects/SSO/Importing_users_from_MediaWiki_to_Firefox_Accounts
 * 
 * @author Renoir Boulanger <renoir@w3.org>
 */

// Password (DO NOT USE THE SAME AS POSTED ONLINE!1)
var password = '4e9bde2d85';

    /**
     * fxa-auth-server listen IP and port
     *
     * Make sure it matches the same configuration
     * otherwise the hashing validation will fail with
     * "Bad mac" error message.
     *
     * See config/config.js in listen section.
     *
     * @type {String} FXA Auth server HTTP endpoint to make calls against
     */
var  instanceEndpoint = 'https://api.accounts.webplatform.org';

var fs = require('fs'),
    publicKey = JSON.parse(fs.readFileSync('../fxa-auth-server/config/public-key.json')),
    Client = require('../fxa-auth-server/client'),
    duration = 1000 * 60 * 60 * 24,
    client = null;

function AccountCreator(dataObject) {
  if ( dataObject[2] === null ) {
    throw 'User MUST have an email address for ' + dataObject[0];
  }
  // Use a hard to guess password to force users to ask a new one
  this.password = password;
  this.email = dataObject[2]; // See _parseUserData
  this.config = {pk: publicKey, d: duration, ep: instanceEndpoint};
  this.userData = this._parseUserData(dataObject);
}

/**
 * Format options array for overloaded signup
 *
 * userData expects 4 members:
 * - MediaWiki username
 * - Full name
 * - Email address
 * - Registration date
 *
 * @param  array  userData      data in a predetermined order
 * @return object optionsObject what we return as 3rd argument to ClientCreate call
 */
AccountCreator.prototype._parseUserData = function _parseUserData(userData) {
    var dateObject = new Date(userData[3]),
        out = {};

    out.preVerified = true;
    out.username = userData[0];
    out.fullName = userData[1] || null;
    out.forceCreatedAt = dateObject.getTime();

    return out;
};

AccountCreator.prototype.makePromiseToCreate = function makePromiseToCreate () {
  return Client.create(this.config.ep, this.email, this.password, this.userData)
          .then(function(a){
            console.log('Created ', a.email);
          });
};

module.exports = AccountCreator;

MediaWiki_SQL_Query.md

Import mediawiki Users

Canonical

• Gist
• Wiki

Process

• Requires NodeJS, NPM
• Clone [https://github.com/webplatform/fxa-auth-server fxa-auth-server] in a folder of the same name
• Install dependencies in fxa-auth-server with npm install
• Go a folder up, create a new one and paste the following in it

Export MediaWiki users

• Run this query from MySQL Workbench.

    SELECT
      CONVERT(user_name USING utf8),
      CONVERT(user_real_name USING utf8),
      CONVERT(user_email USING utf8),
      CONVERT(user_registration, DATETIME),
      user_id
    FROM
      user
    ORDER BY user_id
    LIMIT 40000;

• Export the result in a file users.csv, it’ll be used to import
• Make sure the output is a CSV file ** one entry per line ** Fields in this order: username, realname, email, creation date

package.json

{
  "name": "mediawiki-to-fxa-user-importer",
  "version": "0.0.1",
  "dependencies": {
    "q": "~1.0.0",
    "csv-streamify":"~1.0.0",
    "sleep": "~1.1.5"
  }
}

run.js

/**
 * Import MediaWiki accounts file to run
 *
 * To use within fxa-auth-server deployment.
 *
 * Based on work pushed in https://github.com/mozilla/fxa-auth-server/pull/711
 *
 * See documentation at http://docs.webplatform.org/wiki/WPD:Projects/SSO/Importing_users_from_MediaWiki_to_Firefox_Accounts
 *
 * @author David Kirstein <frozenice@frozenice.de> (http://webplatform.org)
 * @author Renoir Boulanger <renoir@w3.org>
 */

var fs = require('fs');
var Q = require('q');
var csv = require('csv-streamify');
var sleep = require('sleep');
var AccountCreator = require('./accountscreator');

function doSomethingWithLine(line) {
    try {
      var payload = new AccountCreator(line);
    } catch (err) {
      return Q.reject(err);
    }

    return payload.makePromiseToCreate();
}

var fstream = fs.createReadStream('users.csv');
var csvOptions = { empty: null, objectMode: true };
var parser = csv(csvOptions);
var workItems = [], errors = [];

parser.on('readable', function() {
  var line = parser.read(); // line is an array of fields
  sleep.usleep(100 * 1000); // sleep 100 ms
  var promise = doSomethingWithLine(line);
  promise.done(function onFulfilled() {
    // Nothing
  }, function onRejected(a) {
    errors.push([line[0], a.message || a]);
  });
  workItems.push(promise);
});

fstream.pipe(parser).on('end', function() {
  waitForAllPromises();
});

function waitForAllPromises() {
  // use allSettled, all would abort if just one promise would be rejected
  Q.allSettled(workItems).done(function() {
    console.log('all done! errors:');
    console.error(errors);
  });
}

users.csv