SPOJ Crawler

crawler.js

'use strict';

// user=brenolf password='xxxxx' node crawler.js

const USER = process.env.user;
const PASSWORD = process.env.password;

const co = require('co');
const rp = require('request-promise');

const SEARCH_PROBLEMS = /<td[^>]+><a[^>]+>([^<]+)<\/a><\/td>/g;
const SEARCH_ID = /<tr class="kol1"><td class="statustext">\n(\d+)/;

const doIt = function* doIt() {
  const mainCookie = rp.jar();

  const response = yield rp({
    method: 'POST',
    uri: 'http://www.spoj.com/login',
    simple: false,
    resolveWithFullResponse: true,
    followAllRedirects: true,
    form: {
      login_user: USER,
      password: PASSWORD,
      autologin: true
    },
    jar: mainCookie
  });

  const jar = rp.jar();
  const cookie = rp.cookie(mainCookie.getCookieString('http://www.spoj.com'));
  jar.setCookie(cookie, 'http://br.spoj.com');

  const list = yield rp(`http://br.spoj.com/users/${USER}/`);

  let p;

  while (p = SEARCH_PROBLEMS.exec(list)) {
    const submissions = yield rp(`http://br.spoj.com/status/${p[1]},${USER}/`);

    const id = (submissions.match(SEARCH_ID))[1];

    const code = yield rp({
      uri: `http://br.spoj.com/files/src/save/${id}`,
      jar: jar
    });

    return console.log(code);
  }
};

co(doIt)
.catch(err => console.log(err));