Category Sitemaps

const pupeteer = require('puppeteer');
const clients = require('./clients');
const URL = require('url');
const request = require('request');



function delay(time) {
   return new Promise(function(resolve) { 
       setTimeout(resolve, time)
   });
}

	try{
(async () => {
for(const client of clients) {

		console.log(`\n- Client: ${client.name}`);

		const domain = client.url;
		const username = client.username;
		const password = client.password;
		const name = client.name;
	const browser = await pupeteer.launch({args: ['--no-sandbox', '--disable-setuid-sandbox']});
	const page = await browser.newPage();
	page.setUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/601.3.9 (KHTML, like Gecko) Version/9.0.2 Safari/601.3.9");
	await page.setViewport({
  width: 1500,
  height: 2000,
  deviceScaleFactor: 1,
});

await page.goto(`${client.url}/xxx/login.cfm`);
	await page.waitFor('.form-control.sealogininput.seausernameinput');
    
	await page.focus('.form-control.sealogininput.seausernameinput');
	await page.keyboard.type(username);
	await page.focus('.form-control.sealogininput.seapasswordinput');
	await page.keyboard.type(password);
	await page.click('.btn.btn-primary.btn-block.sealoginbutton');
	//await page.waitForNavigation();
	console.log("Log in button clicked");


await page.goto(`${client.url}/xxx/index.cfm`,{ waitUntil: 'networkidle0'});
await page.setViewport({
  width: 1500,
  height: 20000,
  deviceScaleFactor: 1,
});
await delay(5000);
//await page.waitFor('#expandAll');
console.log("expand button arrived");
    
await page.click('#expandAll');

console.log('expand button clicked');
await delay(9000);

console.log('after waiting');
let links = await page.evaluate(() => {
		return Array.from(document.querySelectorAll('.HideCol0PreviewIcon a')).map((val) => val.href);
	});
	console.log(links);

await browser.close();
xmlBegin = "<?xml version=\"1.0\" encoding=\"UTF-8\"?><urlset xmlns=\"https://www.sitemaps.org/schemas/sitemap/0.9\">";
xmlEnd = "</urlset>";
let output = [];
        for (var line = 0; line < links.length; line++) {
           links[line] = "<url><loc>" + links[line] + "</loc></url>" ;
output.push(links[line]);}
let fixedText = output.join("\n");
let finalText = xmlBegin + "\n" + fixedText + "\n" + xmlEnd;

var formData = {body: finalText,name: name};
request.post({
        uri: 'https://cnn.com',
        headers: {
            'Content-Type':'multipart/form-data;charset=UTF-8',
            'Accept-Encoding':'gzip, deflate',
            'X-Requested-With': 'XMLHttpRequest',
            'Accept':'multipart/form-data, text/plain, */*',
            'User-Agent': 'UserAgent' 
        },
        json: false,
        formData: formData
        }, (error, res, body) => {
  if (error) {
    console.error(error)
    return
  }}) //end of post
 }})()
 } catch (err) {
  console.error(err)
}