jnioche/JobBoardNavigationFilter.java

## JobBoardNavigationFilter.java
package com.digitalpebble.crawl;

import java.util.List;

import org.openqa.selenium.By;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.remote.RemoteWebDriver;
import org.openqa.selenium.support.ui.ExpectedConditions;
import org.openqa.selenium.support.ui.WebDriverWait;

import com.digitalpebble.stormcrawler.Metadata;
import com.digitalpebble.stormcrawler.protocol.ProtocolResponse;
import com.digitalpebble.stormcrawler.protocol.selenium.NavigationFilter;

public class JobBoardNavigationFilter extends NavigationFilter {

    @Override
    public ProtocolResponse filter(RemoteWebDriver driver, Metadata metadata) {
        StringBuilder dummyContent = new StringBuilder("<html>");

        // check that we are on the right sort of page
        if (!driver.getCurrentUrl().contains("JobBoard/ListJobs.aspx")) {
            return null;
        }

        // iterate on the result pages
        while (true) {
            // get the links for the current page
            List<WebElement> anchors = driver.findElementsByTagName("A");
            for (WebElement element : anchors) {
                String href = element.getAttribute("href");
                if (!href.contains("JobDetails.aspx"))
                    continue;
                // generate an outlink
                dummyContent.append("<a href=\"").append(href).append("\">");
                dummyContent.append(element.getText()).append("<a>\n");
            }
            // see if there is a clickable 'next page' button
            WebDriverWait wait = new WebDriverWait(driver, 10);
            WebElement nextButton = wait.until(ExpectedConditions
                    .presenceOfElementLocated(By.id("__Next")));
            if (!nextButton.isEnabled())
                break;
            nextButton.click();
        }

        dummyContent.append("</html>");
        return new ProtocolResponse(dummyContent.toString().getBytes(), 200,
                metadata);
    }

}
	package com.digitalpebble.crawl;

	import java.util.List;

	import org.openqa.selenium.By;
	import org.openqa.selenium.WebElement;
	import org.openqa.selenium.remote.RemoteWebDriver;
	import org.openqa.selenium.support.ui.ExpectedConditions;
	import org.openqa.selenium.support.ui.WebDriverWait;

	import com.digitalpebble.stormcrawler.Metadata;
	import com.digitalpebble.stormcrawler.protocol.ProtocolResponse;
	import com.digitalpebble.stormcrawler.protocol.selenium.NavigationFilter;

	public class JobBoardNavigationFilter extends NavigationFilter {

	@Override
	public ProtocolResponse filter(RemoteWebDriver driver, Metadata metadata) {
	StringBuilder dummyContent = new StringBuilder("<html>");

	// check that we are on the right sort of page
	if (!driver.getCurrentUrl().contains("JobBoard/ListJobs.aspx")) {
	return null;
	}

	// iterate on the result pages
	while (true) {
	// get the links for the current page
	List<WebElement> anchors = driver.findElementsByTagName("A");
	for (WebElement element : anchors) {
	String href = element.getAttribute("href");
	if (!href.contains("JobDetails.aspx"))
	continue;
	// generate an outlink
	dummyContent.append("<a href=\"").append(href).append("\">");
	dummyContent.append(element.getText()).append("<a>\n");
	}
	// see if there is a clickable 'next page' button
	WebDriverWait wait = new WebDriverWait(driver, 10);
	WebElement nextButton = wait.until(ExpectedConditions
	.presenceOfElementLocated(By.id("__Next")));
	if (!nextButton.isEnabled())
	break;
	nextButton.click();
	}

	dummyContent.append("</html>");
	return new ProtocolResponse(dummyContent.toString().getBytes(), 200,
	metadata);
	}

	}