shocknawe/scrapper.js

## scrapper.js
// Grab from network call headers
var X_AUTHORIZATION = "2878e4b1-77b0-48f0-bbd2-e25e31fd345b";

var getAllPersonIds = () => {
  var items = document.querySelectorAll("app-network article");
  items = Array.from(items);
  return items.map((item) => {
    let link = item.querySelector(".media-content a[data-test]");
    let itemId = link.getAttribute("data-test");
    itemId = itemId.replace("thingId", "");
    return itemId;
  });
};

var getPersonData = async (personId) => {
  try {
    let url = `https://api-prod.intros.at/1/container/3168/thing/${personId}`;
    let response = await fetch(url, {
      method: "GET",
      headers: {
        "Content-Type": "application/json",
        "login-source": "web",
        "x-authorization": X_AUTHORIZATION,
        "x-grip-version": "Web/7.8.1",
      },
    });
    var data = await response.json();
    var contactData = await getPersonContacts(personId);
    data.data.email = contactData.data.email;
    data.data.phone_number = contactData.data.phone_number;
    return data;
  } catch (err) {
    throw err;
  }
};

var getPersonContacts = async (personId) => {
  try {
    let url = `https://api-prod.intros.at/1/container/3168/thing/${personId}/contact_details`;
    let response = await fetch(url, {
      method: "GET",
      headers: {
        "Content-Type": "application/json",
        "login-source": "web",
        "x-authorization": X_AUTHORIZATION,
        "x-grip-version": "Web/7.8.1",
      },
    });
    var data = await response.json();
    return data;
  } catch (err) {
    throw err;
  }
};

var getAllPersonData = async () => {
  var personIds = getAllPersonIds();
  // var personIds = ["5071963","5103063", "5130480"]; // attendees
  // var personIds = ["5093035","5093059"]; // speakers
  // var personIds = ["5136942"]; // sponsor representative
  let personPromises = personIds.map((id) => getPersonData(id));
  return await Promise.all(personPromises);
};

var cleanString = (str) => {
  let returnStr = str;
  returnStr = returnStr === null || returnStr === undefined ? "" : returnStr;
  returnStr = "" + returnStr;
  return returnStr;
};

var formatItemAsHtmlRow = (item) => {
  let {
    id,
    first_name,
    last_name,
    company_name,
    job_title,
    location,
    email,
    phone_number,
    headline,
    rtm_raw,
  } = item.data;
  let businessCategory = "";
  let businessLine = "";
  let businessCountry = "";

  if (rtm_raw) {
    if (rtm_raw.country_code && rtm_raw.country_code.value) {
      businessCountry = rtm_raw.country_code.value;
    }
    if (rtm_raw.business_category && rtm_raw.business_category.value) {
      businessCategory = rtm_raw.business_category.value.join(", ");
    }
    if (rtm_raw.business_line && rtm_raw.business_line.value) {
      businessLine = rtm_raw.business_line.value.join(", ");
    }
  }

  return `
    <tr>
      <td><a href="https://matchmaking.grip.events/insuretechconnectasia/app/profile/${id}">${id}</a></td>
      <td>${first_name}</td>
      <td>${last_name}</td>
      <td>${company_name}</td>
      <td>${job_title}</td>
      <td>${location}</td>
      <td>${businessCountry}</td>
      <td>${businessCategory}</td>
      <td>${businessLine}</td>
      <td>${cleanString(email)}</td>
      <td>&nbsp;${cleanString(phone_number)}</td>
      <td>${headline}</td>
    </tr>
  `;
};

// copied from https://stackoverflow.com/a/27843359
var exportToExcel = (data) => {
  var htmls = data.map((item) => formatItemAsHtmlRow(item)).join("");

  var header = `
    <tr>
      <td>Link to profile</td>
      <td>First Name</td>
      <td>Last Name</td>
      <td>Company</td>
      <td>Job Title</td>
      <td>Location</td>
      <td>Business Country</td>
      <td>Business Category</td>
      <td>Business Line</td>
      <td>Email</td>
      <td>Phone Number</td>
      <td>Headline</td>
    </tr>`;

  var uri = "data:application/vnd.ms-excel;base64,";
  var template =
    '<html xmlns:o="urn:schemas-microsoft-com:office:office" xmlns:x="urn:schemas-microsoft-com:office:excel" xmlns="http://www.w3.org/TR/REC-html40"><head><!--[if gte mso 9]><xml><x:ExcelWorkbook><x:ExcelWorksheets><x:ExcelWorksheet><x:Name>{worksheet}</x:Name><x:WorksheetOptions><x:DisplayGridlines/></x:WorksheetOptions></x:ExcelWorksheet></x:ExcelWorksheets></x:ExcelWorkbook></xml><![endif]--></head><body><table>{table}</table></body></html>';
  var base64 = function (s) {
    return window.btoa(unescape(encodeURIComponent(s)));
  };

  var format = function (s, c) {
    return s.replace(/{(\w+)}/g, function (m, p) {
      return c[p];
    });
  };

  var ctx = {
    worksheet: "Worksheet",
    table: header + htmls,
  };

  var link = document.createElement("a");
  link.download = "export.xls";
  link.href = uri + base64(format(template, ctx));
  link.click();
};

var scraper = async () => {
  var data = await getAllPersonData();
  exportToExcel(data);
};

scraper();
	// Grab from network call headers
	var X_AUTHORIZATION = "2878e4b1-77b0-48f0-bbd2-e25e31fd345b";

	var getAllPersonIds = () => {
	var items = document.querySelectorAll("app-network article");
	items = Array.from(items);
	return items.map((item) => {
	let link = item.querySelector(".media-content a[data-test]");
	let itemId = link.getAttribute("data-test");
	itemId = itemId.replace("thingId", "");
	return itemId;
	});
	};

	var getPersonData = async (personId) => {
	try {
	let url = `https://api-prod.intros.at/1/container/3168/thing/${personId}`;
	let response = await fetch(url, {
	method: "GET",
	headers: {
	"Content-Type": "application/json",
	"login-source": "web",
	"x-authorization": X_AUTHORIZATION,
	"x-grip-version": "Web/7.8.1",
	},
	});
	var data = await response.json();
	var contactData = await getPersonContacts(personId);
	data.data.email = contactData.data.email;
	data.data.phone_number = contactData.data.phone_number;
	return data;
	} catch (err) {
	throw err;
	}
	};

	var getPersonContacts = async (personId) => {
	try {
	let url = `https://api-prod.intros.at/1/container/3168/thing/${personId}/contact_details`;
	let response = await fetch(url, {
	method: "GET",
	headers: {
	"Content-Type": "application/json",
	"login-source": "web",
	"x-authorization": X_AUTHORIZATION,
	"x-grip-version": "Web/7.8.1",
	},
	});
	var data = await response.json();
	return data;
	} catch (err) {
	throw err;
	}
	};

	var getAllPersonData = async () => {
	var personIds = getAllPersonIds();
	// var personIds = ["5071963","5103063", "5130480"]; // attendees
	// var personIds = ["5093035","5093059"]; // speakers
	// var personIds = ["5136942"]; // sponsor representative
	let personPromises = personIds.map((id) => getPersonData(id));
	return await Promise.all(personPromises);
	};

	var cleanString = (str) => {
	let returnStr = str;
	returnStr = returnStr === null \|\| returnStr === undefined ? "" : returnStr;
	returnStr = "" + returnStr;
	return returnStr;
	};

	var formatItemAsHtmlRow = (item) => {
	let {
	id,
	first_name,
	last_name,
	company_name,
	job_title,
	location,
	email,
	phone_number,
	headline,
	rtm_raw,
	} = item.data;
	let businessCategory = "";
	let businessLine = "";
	let businessCountry = "";

	if (rtm_raw) {
	if (rtm_raw.country_code && rtm_raw.country_code.value) {
	businessCountry = rtm_raw.country_code.value;
	}
	if (rtm_raw.business_category && rtm_raw.business_category.value) {
	businessCategory = rtm_raw.business_category.value.join(", ");
	}
	if (rtm_raw.business_line && rtm_raw.business_line.value) {
	businessLine = rtm_raw.business_line.value.join(", ");
	}
	}

	return `
	<tr>
	<td><a href="https://matchmaking.grip.events/insuretechconnectasia/app/profile/${id}">${id}</a></td>
	<td>${first_name}</td>
	<td>${last_name}</td>
	<td>${company_name}</td>
	<td>${job_title}</td>
	<td>${location}</td>
	<td>${businessCountry}</td>
	<td>${businessCategory}</td>
	<td>${businessLine}</td>
	<td>${cleanString(email)}</td>
	<td> ${cleanString(phone_number)}</td>
	<td>${headline}</td>
	</tr>
	`;
	};

	// copied from https://stackoverflow.com/a/27843359
	var exportToExcel = (data) => {
	var htmls = data.map((item) => formatItemAsHtmlRow(item)).join("");

	var header = `
	<tr>
	<td>Link to profile</td>
	<td>First Name</td>
	<td>Last Name</td>
	<td>Company</td>
	<td>Job Title</td>
	<td>Location</td>
	<td>Business Country</td>
	<td>Business Category</td>
	<td>Business Line</td>
	<td>Email</td>
	<td>Phone Number</td>
	<td>Headline</td>
	</tr>`;

	var uri = "data:application/vnd.ms-excel;base64,";
	var template =
	'<html xmlns:o="urn:schemas-microsoft-com:office:office" xmlns:x="urn:schemas-microsoft-com:office:excel" xmlns="http://www.w3.org/TR/REC-html40"><head><!--[if gte mso 9]><xml><x:ExcelWorkbook><x:ExcelWorksheets><x:ExcelWorksheet><x:Name>{worksheet}</x:Name><x:WorksheetOptions><x:DisplayGridlines/></x:WorksheetOptions></x:ExcelWorksheet></x:ExcelWorksheets></x:ExcelWorkbook></xml><![endif]--></head><body><table>{table}</table></body></html>';
	var base64 = function (s) {
	return window.btoa(unescape(encodeURIComponent(s)));
	};

	var format = function (s, c) {
	return s.replace(/{(\w+)}/g, function (m, p) {
	return c[p];
	});
	};

	var ctx = {
	worksheet: "Worksheet",
	table: header + htmls,
	};

	var link = document.createElement("a");
	link.download = "export.xls";
	link.href = uri + base64(format(template, ctx));
	link.click();
	};

	var scraper = async () => {
	var data = await getAllPersonData();
	exportToExcel(data);
	};

	scraper();