Raidus

sudo apt-get update && sudo apt-get install -yq libgconf-2-4
sudo apt-get update && sudo apt-get install -y wget --no-install-recommends \
    && wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \
    && sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' \
    && apt-get update \
    && apt-get install -y google-chrome-unstable fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst ttf-freefont \
      --no-install-recommends \
    && rm -rf /var/lib/apt/lists/* \
    && apt-get purge --auto-remove -y curl \
    && rm -rf /src/*.deb

Raidus

Motivation

Allow developers to specify workflows without the need for installing additional tooling such as a modeler.

Goals

• It should be easy to define a simple workflow with a text editor only
• The format must map down directly to BPMN
• The format must be based on an existing text based format like (XML, JSON, YAML, TOML, ...)

Raidus

// Reference: https://blog.apify.com/how-to-make-headless-chrome-and-puppeteer-use-a-proxy-server-with-authentication-249a21a79212

const puppeteer = require('puppeteer');
const proxyChain = require('proxy-chain');

const { PROXYMESH_USER, PROXYMESH_PASSWORD } = require('../config/keys');

(async () => {
  const oldProxyUrl = `http://${PROXYMESH_USER}:${PROXYMESH_PASSWORD}@de.proxymesh.com:31280`;
  const newProxyUrl = await proxyChain.anonymizeProxy(oldProxyUrl);

Raidus

const request = require('request-promise');
const puppeteer = require('puppeteer');
const fs = require('fs');

const Sock5HttpAgent = require('socks5-http-client/lib/Agent');
const Sock5HttpsAgent = require('socks5-https-client/lib/Agent');

const logDir = 'tmp';
if (!fs.existsSync(logDir)) {
  fs.mkdirSync(logDir);

Raidus

// To do: Get more agents from: https://developers.whatismybrowser.com/useragents/explore/software_name/chrome/1

const UAs = [
  'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36',
  'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36',
  'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36',
  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36',
  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/603.2.4 (KHTML, like Gecko) Version/10.1.1 Safari/603.2.4',
  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36',
  'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36',

Raidus

const puppeteer = require('puppeteer');
const ProxyChain = require('proxy-chain');

const ROUTER_PROXY = 'http://127.0.0.1:8000';

const uas = [
  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36',
  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36',
];

Raidus

// Returns an array of dates between the two dates
var getDates = function(startDate, endDate) {
  var dates = [],
      currentDate = startDate,
      addDays = function(days) {
        var date = new Date(this.valueOf());
        date.setDate(date.getDate() + days);
        return date;
      };
  while (currentDate <= endDate) {

Raidus

const winston = require('winston');
const fs = require('fs');
const path = require('path');

const dir = path.resolve(__dirname, '../logs/');

if (!fs.existsSync(dir)) {
  fs.mkdirSync(dir);
}

Raidus

var exec = require('child_process').exec,
    url = "http://google.com/",
    timeout = "3",
    data="?q=test";

var time = process.hrtime();
exec('curl --max-time ' + timeout + ' -d \'' + data + '\' '  + url, function (error, stdout, stderr) {
    var diff = process.hrtime(time);
    //console.log('stdout: ' + stdout);
    //console.log('stderr: ' + stderr);

Raidus

/**
 * Install and enable the rabbitmq_delayed_message_exchange plugin as described by Alvaro Videla in this blogpost:
 * https://www.rabbitmq.com/blog/2015/04/16/scheduling-messages-with-rabbitmq/
 */
const amqp = require('amqplib');
const exchange = 'yourExchangeName';
const queue = 'yourQueueName';
const queueBinding = 'yourQueueBindingName';

// Message consumer