joker.js

var identities = [];

db.collection("identities").doc("jokerstash").collection("queue").get().
	then(snapshot => {
		snapshot.forEach(doc => {
			var newIdentity = {
				status: doc.data().status,
				url: doc.data().url,
				username: doc.data().username,
				pwd: doc.data().pwd,
				captcha: doc.data().captcha,
				extract: doc.data().extract,
				baseids: doc.data().baseids
			}
			stuff.push(newIdentity);
			console.log(stuff);
		});
		return Promise.resolve(stuff);
	})
	.then(stuff => {
		// this will wait for every item in stuff to resolve using the joker() function
		return Promise.all(
			stuff.map(i => joker(i))
		)
	}) 
	.then(() => {
		console.log('Done mapping through all the stuff!');
	})
	.catch(reason => {
		console.log(reason);
	})




async function joker(stuff) {

	console.log(stuff.url);

	var driver = await new webdriver.Builder().forBrowser('chrome').build();
	var loginMap = await new Map();
	await loginMap.set('name', 'jstash');
	await loginMap.set('url', stuff.url.toString());
	await loginMap.set('captchaAttempts', stuff.captcha);
	await loginMap.set('userNameEdit', locators.usernameEdit);
	await loginMap.set('passwordEdit', locators.passwordEdit);
	await loginMap.set('captchaImage', locators.capthaImage);
	await loginMap.set('captchaEdit', locators.captchaEdit);
	await loginMap.set('loginButton', locators.loginButton);
	await loginMap.set('confirmationElement', locators.userNameHeader);
	await loginMap.set('userNameVal', stuff.username);
	await loginMap.set('passwordVal', stuff.pwd);

	await utility.login(driver, loginMap);

	//iterate through baseids
	for (var i = 0, len = baseids.length; i < len; i++) {

		// Check if string or number is supplied
		var data = await parseInt(baseids[i], 10);
		if (isNaN(data)) {
			//base is new, and does not yet have an id available for us to filter on
			await driver.get(url + extract);
			await driver.sleep(5000);

			//select the base & apply the filters
			await utility.selectByPartialText(driver, locators.baseFilter, utility.GlobalTimeout, baseids[i]);
		}
		else {
			var filterurl = await url + extract + '?base=' + baseids[i];
			await driver.get(filterurl);
			await driver.sleep(3000);
		}



		var fileName = outputPath + 'Jstash_' + extract + '_' + baseids[i] + '_' + (new Date).getTime().toString() + '.html';

		//move to appropriate page
		var instance_url = url + extract + "?base=" + baseids[i];
		var cart_url = url + "cart";

		await logger.info('================================');
		await logger.info("Instance url: " + instance_url);
		await driver.get(instance_url);
		var pageCounter = 0;
		do {
			//add the page to cart
			await utility.clickElement(driver, locators.addAllItemsToCart, 5000);
			await driver.sleep(8000);

			//continue paginating
			await utility.clickElement(driver, locators.nextButton, 5000);

			//keep us below 50
			await pageCounter++;
			if (pageCounter >= 49) {
				break;
			}

		}
		while (await utility.verifyElement(driver, locators.nextButton, 5000));

		//proceed to cart
		await driver.sleep(2000);
		await logger.info('Proceeding to cart...');
		await driver.get(cart_url);


		var data = await driver.getPageSource();


		//push data to firestore 
		var ts = await (new Date).getTime().toString();
		var dataRef = db.collection('data').doc('output').collection(extract);
		var setData = dataRef.add({
			baseid: baseids[i],
			html: data,
			timestamp: ts
		})
		//clear cart
		await clearCart(driver);
		await logger.info('Successful execution for base ' + baseids[i]);
		await logger.info('Assimilated ' + (pageCounter * 100).toString() + ' new records.');
		await logger.info('================================');

	}
}

/* removes all of our cart contents */
async function clearCart(driverVal) {

	var flag = await utility.verifyElement(driverVal, locators.clearCartButton, 5000);

	if (flag) {
		await utility.clickElement(driverVal, locators.clearCartButton, utility.GlobalTimeout);
		await driverVal.sleep(1000);
		await driverVal.switchTo().alert().accept();
		await driverVal.sleep(10000);
	}
}