Skip to content

Instantly share code, notes, and snippets.

@caseyscarborough
Last active August 29, 2015 14:00
Show Gist options
  • Save caseyscarborough/3abd61d2780fa287291c to your computer and use it in GitHub Desktop.
Save caseyscarborough/3abd61d2780fa287291c to your computer and use it in GitHub Desktop.
A Java program I've written used to parse a test bank on Desire2Learn using Selenium and Jsoup, and a Groovy script that sorts the questions, and removes all the duplicates. Not meant to be efficient, just to get all questions from the test bank, an otherwise VERY tedious task.
import org.apache.commons.lang3.StringEscapeUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.safety.Whitelist;
import org.jsoup.select.Elements;
import org.openqa.selenium.By;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.firefox.FirefoxDriver;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.util.ArrayList;
import java.util.Collections;
public class D2LRipper {
public static void main(String[] args) {
WebDriver driver = new FirefoxDriver();
driver.get("Desire2Learn Login URL");
WebElement usernameField = driver.findElement(By.id("Username"));
WebElement passwordField = driver.findElement(By.id("Password"));
WebElement loginButton = driver.findElement(By.name("Login"));
usernameField.sendKeys("username");
passwordField.sendKeys("password");
loginButton.click();
try {
for (int i = 0; i < 100; i++) {
driver.get("URL for 5 question self assessment");
Thread.sleep(2000);
StringBuilder output = new StringBuilder();
String content;
File file = new File("output.md");
if (!file.exists()) {
file.createNewFile();
}
FileWriter fw = new FileWriter(file.getAbsoluteFile(), true);
BufferedWriter bw = new BufferedWriter(fw);
content = driver.getPageSource();
Document doc = Jsoup.parse(content, "Root URL");
Elements elements = doc.select(".drt");
for (int h = 0; h < elements.size(); h += elements.size() / 5) {
output.append(StringEscapeUtils.unescapeHtml4(Jsoup.clean(elements.get(h).text(), Whitelist.none())));
output.append("\n");
ArrayList<String> answers = new ArrayList<String>();
for (int j = h + 1; j < h + 5; j++) {
answers.add(Jsoup.clean(elements.get(j).text(), Whitelist.none()));
}
Collections.sort(answers);
for (String s : answers) {
output.append("\n* ");
output.append(StringEscapeUtils.unescapeHtml4(s));
}
output.append("\n\n");
}
bw.write(output.toString());
bw.close();
}
} catch (Exception e) {
System.out.println("SHIT");
e.printStackTrace();
}
driver.quit();
}
}
def file = new File("output.md")
def contents = file.readLines()
def questions = []
print "Reading questions from file..."
for (int i = 0; i < contents.size(); i += 7) {
def question = new StringBuilder()
for (int j = 0; j < 7; j++) {
question << contents.get(j + i) + "\n";
}
questions << question.toString()
}
println "done."
print "Sorting all questions..."
questions.sort { it.substring(0, 30) }
println "done."
print "Removing duplicate questions..."
questions.unique { a, b -> a <=> b }
println "done."
println "There are a total of ${questions.size()} unique questions."
new File("sorted.md").withWriter { out ->
questions.each() { q ->
out.write(q)
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment