This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
const request = require('request-promise'); | |
const puppeteer = require('puppeteer'); | |
const fs = require('fs'); | |
const Sock5HttpAgent = require('socks5-http-client/lib/Agent'); | |
const Sock5HttpsAgent = require('socks5-https-client/lib/Agent'); | |
const logDir = 'tmp'; | |
if (!fs.existsSync(logDir)) { | |
fs.mkdirSync(logDir); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// To do: Get more agents from: https://developers.whatismybrowser.com/useragents/explore/software_name/chrome/1 | |
const UAs = [ | |
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36', | |
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36', | |
'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36', | |
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36', | |
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/603.2.4 (KHTML, like Gecko) Version/10.1.1 Safari/603.2.4', | |
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36', | |
'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36', |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
sudo apt-get update && sudo apt-get install -yq libgconf-2-4 | |
sudo apt-get update && sudo apt-get install -y wget --no-install-recommends \ | |
&& wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \ | |
&& sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' \ | |
&& apt-get update \ | |
&& apt-get install -y google-chrome-unstable fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst ttf-freefont \ | |
--no-install-recommends \ | |
&& rm -rf /var/lib/apt/lists/* \ | |
&& apt-get purge --auto-remove -y curl \ | |
&& rm -rf /src/*.deb |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
const puppeteer = require('puppeteer'); | |
const ProxyChain = require('proxy-chain'); | |
const ROUTER_PROXY = 'http://127.0.0.1:8000'; | |
const uas = [ | |
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36', | |
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36', | |
]; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// Returns an array of dates between the two dates | |
var getDates = function(startDate, endDate) { | |
var dates = [], | |
currentDate = startDate, | |
addDays = function(days) { | |
var date = new Date(this.valueOf()); | |
date.setDate(date.getDate() + days); | |
return date; | |
}; | |
while (currentDate <= endDate) { |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
const winston = require('winston'); | |
const fs = require('fs'); | |
const path = require('path'); | |
const dir = path.resolve(__dirname, '../logs/'); | |
if (!fs.existsSync(dir)) { | |
fs.mkdirSync(dir); | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
var exec = require('child_process').exec, | |
url = "http://google.com/", | |
timeout = "3", | |
data="?q=test"; | |
var time = process.hrtime(); | |
exec('curl --max-time ' + timeout + ' -d \'' + data + '\' ' + url, function (error, stdout, stderr) { | |
var diff = process.hrtime(time); | |
//console.log('stdout: ' + stdout); | |
//console.log('stderr: ' + stderr); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
/** | |
* Install and enable the rabbitmq_delayed_message_exchange plugin as described by Alvaro Videla in this blogpost: | |
* https://www.rabbitmq.com/blog/2015/04/16/scheduling-messages-with-rabbitmq/ | |
*/ | |
const amqp = require('amqplib'); | |
const exchange = 'yourExchangeName'; | |
const queue = 'yourQueueName'; | |
const queueBinding = 'yourQueueBindingName'; | |
// Message consumer |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
require('dotenv').config() | |
const cors = require('cors') | |
const bodyParser = require('body-parser') | |
const express = require('express') | |
const expressJwt = require('express-jwt') | |
const cookieSession = require('cookie-session') | |
const jwt = require('jsonwebtoken') | |
const passport = require('passport') | |
const GoogleStrategy = require('passport-google-oauth20').Strategy | |
const jwtSecret = Buffer.from('Zn8Q5tyZ/G1MHltc4F/gTkVJMlrbKiZt', 'base64') |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Redis Cheatsheet | |
# All the commands you need to know | |
redis-server /path/redis.conf # start redis with the related configuration file | |
redis-cli # opens a redis prompt | |
# Strings. |