Last active
January 26, 2021 12:02
-
-
Save onmax/18e1ee9afab338d5db376388e122be17 to your computer and use it in GitHub Desktop.
Automated information gathering from DAAD webpage to a CSV format
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
let zip = (...rows) => [...rows[0]].map((_,c) => rows.map(row => row[c])) | |
/* Gets all info from one page in a CSV format */ | |
function getCSVpage() { | |
let divs = Array.from(document.querySelectorAll(".view-tab li > div")) | |
const names = divs.map(d => d.querySelector("h3 strong").outerText).map(m => m.replace(",",".")) | |
const links = divs.map(d => d.querySelector("h3 a").href) | |
const unis = divs.map(d => d.querySelector("h3 span").outerText).map(m => m.replace(",",".")) | |
const tuition = divs.map(d => d.querySelector("table tbody tr:nth-child(2) td:last-child").outerText).map(m => m.replace(",",".")) | |
const admision = divs.map(d => d.querySelector("table tbody tr:nth-child(4) td:last-child").outerText) | |
return zip(names, unis, tuition, links, admision).map(m => `${m[0]},${m[1]},${m[2]},${m[3]},${m[4]}`).join("\n") | |
} | |
/* Converts data to CSV file */ | |
function toCSV() { | |
const element = document.createElement("a"); | |
const header = "name, university, tuition, link, admision" | |
element.setAttribute( | |
"href", | |
`data:text/csv;charset=utf-8,${header}\n${csv}\n` | |
); | |
element.setAttribute("download", "data.csv"); | |
element.style.display = "none"; | |
document.body.appendChild(element); | |
element.click(); | |
} | |
/* Loops for every page */ | |
function getCsv() { | |
csv += "\n" + getCSVpage() | |
btn = document.querySelector(".pagination li.next a:not(.disabled)") | |
if (btn === null) return toCSV() | |
btn.click() | |
setTimeout(() => getCsv(), 1000) | |
} | |
let csv = "" | |
getCsv() | |
// It will download a file named data.csv |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
An example for this link: