Hamcrest matcher to take a screenshot from a running selenium WebDriver and compare it to a known good * "reference image". One of the advancement we still need is to inject custom css (or js) before taking a screenshot to cover dynamic elements that should not fail the test (like a stock ticker or weather report)

PerceptualDiffMatcher.java

package com.redheap.test.perceptual;

import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.StringWriter;

import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;

import java.util.ArrayList;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;

import org.apache.commons.io.IOUtils;

import org.hamcrest.Description;
import org.hamcrest.Factory;
import org.hamcrest.Matcher;
import org.hamcrest.TypeSafeMatcher;

import org.openqa.selenium.OutputType;
import org.openqa.selenium.TakesScreenshot;

/**
 * Hamcrest matcher to take a screenshot from a running selenium WebDriver and compare it to a known good
 * "reference image".<p>
 * The path to the expected ("known good") image is given to the constructor of this PerceptualDiffMatcher. When
 * this matcher is used in an assertion with a selenium webdriver that implements org.openqa.selenium.TakesScreenshot
 * it will take a screenshot of that driver in its current state and compare it to the known good image. If the
 * two are exactly the same the match succeeds and no files will be saved to disk.<br>
 * When a difference is found between the known good image and the current screenshot, two additional files will be
 * created. One with the current screenshot (and suffix -actual) and one that highlights the differences (and
 * -diff suffix). So when the reference image is test01.png this will create test01-actual.png and test01-diff.png
 * in the same directory as the test01.png file.<br>
 * When the known good image does not (yet) exist the matcher will also fail, but leave the actual screenshot on disk.
 * This can be manually verified and renamed to the name of the known good image. For example is the (non existing)
 * known image is named test01.png this matcher will create a test01-actual.png that can be manually renamed.
 * <p>
 * Example usage:<br>
 * <pre>
 *   import static nl.rechtspraak.mwo.test.perceptual.PerceptualDiffMatcher.*;
 *   import static org.junit.Assert.*;
 *   import java.nio.file.Paths;
 *
 *   assertThat(driver, hasSimilarScreenshot(Paths.get("expectations/happy-01.png")));
 * </pre>
 */
@SuppressWarnings("oracle.jdeveloper.java.tag-is-misplaced") // because jdev complains about @value tags
public class PerceptualDiffMatcher extends TypeSafeMatcher<TakesScreenshot> {

    private static final Logger log = Logger.getLogger(PerceptualDiffMatcher.class.getName());
    private static final String SYSPROP_NAME = "imagemagick.compare.executable";

    private static final String SUFFIX_ACTUAL = "-actual";
    private static final String SUFFIX_DIFF = "-diff";

    private static final int COMPARE_RESULT_NODIFF = 0;
    private static final int COMPARE_RESULT_DIFF = 1;
    private static final int COMPARE_RESULT_ERROR = 2;

    private final Path referenceImage;

    /**
     * Constructor to create a new PerceptualDiffMatcher.
     * @param referenceImage path to a png file with the reference ("known good") image that we need to compare
     * the current view to. Can be an absolute path or a relative path which will be resolved from the current
     * working directory.
     */
    public PerceptualDiffMatcher(final Path referenceImage) {
        this.referenceImage = referenceImage;
    }

    /**
     * Perform the match by taking a screenshot from the given webdriver and comparing it to the known good image
     * supplied to the constructor. Could create two additional files in the same directory as the given known
     * good image. One with the {@value #SUFFIX_ACTUAL} suffix with a screenshot of the current state of the webdriver
     * and one with the {@value #SUFFIX_DIFF} suffix that hightlights the differences between the known good image and
     * the current screenshot.
     * @param driver webdriver to take a screenshot of to compare to the known good image
     * @return {@code true} if the current screenshot is exactly the same as the known good image supplied to the
     * constructor, otherwise {@code false} which could mean the screenshot differs from the known good image, the
     * known good image doesn't exist yet, or some unexpected error occured.
     */
    @Override
    protected boolean matchesSafely(TakesScreenshot driver) {
        try {
            takeScreenshot(driver);
            if (!Files.isReadable(referenceImage)) {
                return false;
            }
            int result = runDiff();
            switch (result) {
            case COMPARE_RESULT_NODIFF:
                Files.deleteIfExists(buildActualPath());
                Files.deleteIfExists(buildDiffPath());
                return true;
            case COMPARE_RESULT_DIFF:
                return false; // keep screenshot and diff image
            case COMPARE_RESULT_ERROR:
                Files.deleteIfExists(buildDiffPath());
                return false;
            default:
                throw new AssertionError("unexpected result from compare: " + result);
            }
        } catch (IOException | InterruptedException e) {
            try {
                Files.deleteIfExists(buildActualPath());
                Files.deleteIfExists(buildDiffPath());
            } catch (IOException f) {
                f.hashCode(); // ignore exception while deleting files
            }
            throw new AssertionError(e);
        }
    }

    /**
     * Build a path to the file to be used for the actual (aka current) screenshot. This is the path to the
     * known good image supplied to the constructor, with {@value SUFFIX_ACTUAL} added to the filename itself.
     * For example, {@code tests/test01.png} will be translated to {@code tests/test01-actual.png}
     * @return path to the file to be used to the current screenshot
     */
    protected Path buildActualPath() {
        return siblingWithSuffix(referenceImage, SUFFIX_ACTUAL);
    }

    /**
     * Build a path to the file to be used for the difference between the actual and the known good screenshot. This is
     * the path to the known good image supplied to the constructor, with {@value SUFFIX_DIFF} added to the filename
     * itself. For example, {@code tests/test01.png} will be translated to {@code tests/test01-diff.png}
     * @return path to the file to be used for the difference between the current and the known good screenshot
     */
    protected Path buildDiffPath() {
        return siblingWithSuffix(referenceImage, SUFFIX_DIFF);
    }

    private Path siblingWithSuffix(final Path base, final String suffix) {
        String filename = base.getFileName().toString();
        String basename = filename.substring(0, filename.lastIndexOf("."));
        String extension = filename.substring(filename.lastIndexOf(".") + 1);
        Path retval = base.resolveSibling(basename + suffix + "." + extension);
        return retval;
    }

    /**
     * Take a screenshot of the current webdriver screen to the path determined by {@link #buildActualPath}
     * @param driver webdriver to take the screenshot from
     * @throws IOException
     */
    private void takeScreenshot(TakesScreenshot driver) throws IOException {
        Path file = buildActualPath();
        byte[] bytes = driver.getScreenshotAs(OutputType.BYTES);
        Files.copy(new ByteArrayInputStream(bytes), file, StandardCopyOption.REPLACE_EXISTING);
    }

    /**
     * Run ImageMagick compare tool to compare known good (aka reference) screenshot with the actual screenshot.
     * @return 0 if no diff was found, 1 if a diff was found and saved to disk or 2 if an error occured
     * @throws IOException
     * @throws InterruptedException
     * @see #buildActualPath
     */
    private int runDiff() throws IOException, InterruptedException {
        String[] cmd = buildDiffArgs().toArray(new String[0]);
        Process exec = Runtime.getRuntime().exec(cmd);
        if (log.isLoggable(Level.FINE)) {
            StringWriter stderr = new StringWriter();
            IOUtils.copy(exec.getErrorStream(), stderr);
            log.fine("ImageMagick compare output:\n" + stderr);
        }
        return exec.waitFor();
    }

    /**
     * Build the command line arguments to invoke ImageMagick's compare tool to compare the actual screenshot with
     * the expected (known good) image.
     * @return list of command arguments where the first element is the location of the actual executable and all
     * other elements are command line arguments
     * @throws IOException
     */
    protected List<String> buildDiffArgs() throws IOException {
        List<String> cmd = new ArrayList<>();
        cmd.add(findImageMagick().toRealPath().toString());
        cmd.add("-verbose");
        cmd.add("-metric");
        cmd.add("RMSE");
        cmd.add("-highlight-color");
        cmd.add("Red");
        cmd.add(referenceImage.toRealPath().toString());
        cmd.add(buildActualPath().toRealPath().toString());
        cmd.add(buildDiffPath().toAbsolutePath().normalize().toString());
        return cmd;
    }

    /**
     * Determine the location of ImageMagick's compare executable.
     * @return the value of system property {@value #SYSPROP_NAME}, or if that does not exist the value of the
     * system environment variable with the same name, or throwing an AssertionError if both are undefined
     * @throws AssertionError when both the system property and environment variable are unknown
     */
    protected Path findImageMagick() {
        String executable = System.getProperty(SYSPROP_NAME);
        if (executable == null) {
            executable = System.getenv(SYSPROP_NAME);
        }
        if (executable == null) {
            throw new AssertionError("specify location of compare tool in system property or environment variable " +
                                     SYSPROP_NAME);
        }
        return Paths.get(executable);
    }

    /**
     * Describe what this matcher would be expecting as input.
     * @param description
     */
    @Override
    public void describeTo(Description description) {
        description.appendText("a screenshot visually similar to ").appendText(referenceImage.toAbsolutePath().normalize().toString());
    }

    /**
     * Describe the mismatch explaining what was actually encountered compared to the excepted value.
     * @param driver webdriver we took the screenshot from
     * @param mismatchDescription
     */
    @Override
    protected void describeMismatchSafely(TakesScreenshot driver, Description mismatchDescription) {
        if (!Files.isReadable(referenceImage)) {
            mismatchDescription.appendText("reference image not found, actual screenshot saved to ").appendText(buildActualPath().toAbsolutePath().normalize().toString());
            return;
        }
        try {
            mismatchDescription.appendText("was screenshot saved at ");
            mismatchDescription.appendText(buildActualPath().toRealPath().toString());
            Path diff = buildDiffPath();
            if (Files.isReadable(diff)) {
                mismatchDescription.appendText(" with differences marked in ");
                mismatchDescription.appendText(diff.toRealPath().toString());
            }
        } catch (IOException e) {
            throw new AssertionError(e);
        }
    }

    /**
     * Static factory method to create a PerceptualDiffMatcher instance.
     * @param referenceImage path to the known good (aka expected) image
     * @return a PerceptualDiffMatcher
     */
    @Factory
    public static Matcher<TakesScreenshot> hasSimilarScreenshot(Path referenceImage) {
        return new PerceptualDiffMatcher(referenceImage);
    }

}