Skip to content

Instantly share code, notes, and snippets.

@onmax
Last active January 26, 2021 12:02
Show Gist options
  • Save onmax/18e1ee9afab338d5db376388e122be17 to your computer and use it in GitHub Desktop.
Save onmax/18e1ee9afab338d5db376388e122be17 to your computer and use it in GitHub Desktop.
Automated information gathering from DAAD webpage to a CSV format
let zip = (...rows) => [...rows[0]].map((_,c) => rows.map(row => row[c]))
/* Gets all info from one page in a CSV format */
function getCSVpage() {
let divs = Array.from(document.querySelectorAll(".view-tab li > div"))
const names = divs.map(d => d.querySelector("h3 strong").outerText).map(m => m.replace(",","."))
const links = divs.map(d => d.querySelector("h3 a").href)
const unis = divs.map(d => d.querySelector("h3 span").outerText).map(m => m.replace(",","."))
const tuition = divs.map(d => d.querySelector("table tbody tr:nth-child(2) td:last-child").outerText).map(m => m.replace(",","."))
const admision = divs.map(d => d.querySelector("table tbody tr:nth-child(4) td:last-child").outerText)
return zip(names, unis, tuition, links, admision).map(m => `${m[0]},${m[1]},${m[2]},${m[3]},${m[4]}`).join("\n")
}
/* Converts data to CSV file */
function toCSV() {
const element = document.createElement("a");
const header = "name, university, tuition, link, admision"
element.setAttribute(
"href",
`data:text/csv;charset=utf-8,${header}\n${csv}\n`
);
element.setAttribute("download", "data.csv");
element.style.display = "none";
document.body.appendChild(element);
element.click();
}
/* Loops for every page */
function getCsv() {
csv += "\n" + getCSVpage()
btn = document.querySelector(".pagination li.next a:not(.disabled)")
if (btn === null) return toCSV()
btn.click()
setTimeout(() => getCsv(), 1000)
}
let csv = ""
getCsv()
// It will download a file named data.csv
@onmax
Copy link
Author

onmax commented Jan 26, 2021

An example for this link:

image

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment