package com.bazaarvoice.mbogner.utils;

import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpMethod;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;

import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathFactory;
import java.io.ByteArrayInputStream;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Set;

/**
 * @author: Matthew Bogner
 * @date: 10/25/11
 */
public class WebPageTestRunner {

    public static HttpClient client = new HttpClient();
    public static XPath xpath = XPathFactory.newInstance().newXPath();

    public static Set<String> hostnameFilter = new HashSet<String>();

    public static void main(String[] args) throws Exception {
        hostnameFilter.add("cdn.api.twitter.com");
        hostnameFilter.add("platform.twitter.com");

        PostMethod post = new PostMethod("http://www.webpagetest.org/runtest.php");
        post.addParameter("url", "www.aol.com");
        post.addParameter("runs", "1");                  // Only request the page once
        post.addParameter("fvonly", "1");                // Only look at the FirstView
        post.addParameter("location", "Dulles_IE7.DSL"); // Dulles, VA with IE7 and DSL connection
        post.addParameter("f", "xml");                   // Respond with XML
        post.addParameter("k", args[0]);                 // API Key from WebPageTest.org

        // Now we need to (optionally) send over a script so that we can
        // override the DNS entries for certain hosts that the page will
        // attempt to reverence.

        //post.addParameter("script",
        //        "navigate http://www.aol.com\n" +
        //        "setDns platform.twitter.com 123.456.789.2\n" +
        //        "setDns cdn.api.twitter.com 123.456.789.2");

        String responseBody = executeHttpMethod(post);

        System.out.println(responseBody);

        Node statusCodeNode = (Node) xpath.evaluate("/response/statusCode", getXmlSrc(responseBody), XPathConstants.NODE);
        String statusCode = statusCodeNode.getTextContent();

        System.out.println("StatusCode = " + statusCode + "\n");
        if ("200".equals(statusCode)) {
            // Request was successful.  Wait for the test to complete.
            Node testIdNode = (Node) xpath.evaluate("/response/data/testId", getXmlSrc(responseBody), XPathConstants.NODE);

            waitForTestCompletion(testIdNode.getTextContent());
        }


    }

    private static InputSource getXmlSrc(String content) throws Exception {
        return new InputSource(new ByteArrayInputStream(content.getBytes("UTF-8")));
    }

    public static String executeHttpMethod(HttpMethod method) throws Exception {
        int responseCode;
        String responseBody;
        try {
            responseCode = client.executeMethod(method);
            responseBody = IOUtils.toString(method.getResponseBodyAsStream());
        } finally {
            method.releaseConnection();
        }

        if (responseCode != 200) {
            throw new Exception("Invalid server response. \nResponse code: " + responseCode + "\nResponse body: " + responseBody);
        }

        return responseBody;
    }
    
    private static void waitForTestCompletion(String testId) throws Exception {
        PostMethod post = new PostMethod("http://www.webpagetest.org/testStatus.php");
        post.addParameter("f", "xml");                      // Respond with XML
        post.addParameter("test", testId);

        String responseBody = executeHttpMethod(post);

        Node statusCodeNode = (Node) xpath.evaluate("/response/statusCode", getXmlSrc(responseBody), XPathConstants.NODE);
        String statusCode = statusCodeNode.getTextContent();
        // 200 indicates test is completed. 1XX means the test is still in progress. And 4XX indicates some error.

        if (statusCode.startsWith("4")) {
            System.err.println(responseBody);
            throw new Exception("Error getting test results.");
        } else if (statusCode.startsWith("1")) {
            System.out.println("Test not completed.  Waiting for 30 seconds and retrying...");
            Thread.sleep(30000); // Wait for 30sec
            waitForTestCompletion(testId);
        } else if ("200".equals(statusCode)) {
            obtainTestResults(testId);
        } else {
            System.err.println(responseBody);
            throw new Exception("Unknown statusCode in response");
        }
    }

    private static void obtainTestResults(String testId) throws Exception {
        GetMethod get = new GetMethod("http://www.webpagetest.org/xmlResult/" + testId + "/");

        String responseBody = executeHttpMethod(get);
        Node statusCodeNode = (Node) xpath.evaluate("/response/statusCode", getXmlSrc(responseBody), XPathConstants.NODE);
        String statusCode = statusCodeNode.getTextContent();
        if (!"200".equals(statusCode)) {
            System.err.println(responseBody);
            throw new Exception("Unable to obtain raw test results");
        }


        NodeList requestsDataUrlNodes = (NodeList) xpath.evaluate("/response/data/run/firstView/rawData/requestsData",
                                                                  getXmlSrc(responseBody),
                                                                  XPathConstants.NODESET);
        for(int nodeCtr = 0; nodeCtr < requestsDataUrlNodes.getLength(); ++nodeCtr) {
            Node requestsDataNode = requestsDataUrlNodes.item(nodeCtr);
            String requestsDataUrl = requestsDataNode.getTextContent().trim();
            analyzeTestResult(requestsDataUrl);
        }
    }

    private static void analyzeTestResult(String requestsDataUrl) throws Exception {
        System.out.println("\n\nAnalyzing results for " + requestsDataUrl);

        /*
        Things we want to track for each hostname.

        Total # requests
        Total # of requests for each content type
        Total # of bytes for each content type
        Total Time to First Byte
        Total DNS Time
        Total bytes
        Total connection time
         */

        HashMap<String, Integer> numRequestsPerHost = new HashMap<String, Integer>();
        HashMap<String, HashMap<String, Integer>> numRequestsPerHostPerContentType = new HashMap<String, HashMap<String, Integer>>();

        HashMap<String, Integer> totalTTFBPerHost = new HashMap<String, Integer>();
        HashMap<String, Integer> totalDNSLookupPerHost = new HashMap<String, Integer>();
        HashMap<String, Integer> totalInitialCnxnTimePerHost = new HashMap<String, Integer>();

        HashMap<String, HashMap<String, Integer>> totalBytesPerHostPerContentType = new HashMap<String, HashMap<String, Integer>>();
        HashMap<String, Integer> totalBytesPerHost = new HashMap<String, Integer>();


        String responseBody = executeHttpMethod(new GetMethod(requestsDataUrl)); // Unlike the rest, this response will be tab-delimited
        String[] lines = StringUtils.split(responseBody, "\n");

        for (int lineCtr = 1; lineCtr < lines.length; ++lineCtr) {
            String line = lines[lineCtr];
            String[] columns = StringUtils.splitPreserveAllTokens(line, "\t");

            String hostname = columns[5];
            String contentType = columns[18];
            String ttfb = StringUtils.isBlank(columns[9]) ? "0" : columns[9];
            String dns = StringUtils.isBlank(columns[47]) ? "0" : columns[47];
            String cnxn = StringUtils.isBlank(columns[48]) ? "0" : columns[48];
            String bytes = StringUtils.isBlank(columns[13]) ? "0" : columns[13];
            

            if ("0".equals(bytes) || (!hostnameFilter.isEmpty() && !hostnameFilter.contains(hostname))) {
                continue;
            }


            // Track total # requests per host
            if (!numRequestsPerHost.containsKey(hostname)) {
                numRequestsPerHost.put(hostname, new Integer(1));
            } else {
                numRequestsPerHost.put(hostname, numRequestsPerHost.get(hostname) + 1);
            }

            // Track total # requests per host per content-type
            if (!numRequestsPerHostPerContentType.containsKey(hostname)) {
                HashMap<String, Integer> tmp = new HashMap<String, Integer>();
                tmp.put(contentType, new Integer(1));
                numRequestsPerHostPerContentType.put(hostname, tmp);
            } else if (!numRequestsPerHostPerContentType.get(hostname).containsKey(contentType)) {
                numRequestsPerHostPerContentType.get(hostname).put(contentType, new Integer(1));
            } else {
                numRequestsPerHostPerContentType.get(hostname).put(contentType, numRequestsPerHostPerContentType.get(hostname).get(contentType) + 1);
            }

            // Track total # bytes per host per content-type
            if (!totalBytesPerHostPerContentType.containsKey(hostname)) {
                HashMap<String, Integer> tmp = new HashMap<String, Integer>();
                tmp.put(contentType, Integer.valueOf(bytes));
                totalBytesPerHostPerContentType.put(hostname, tmp);
            } else if (!totalBytesPerHostPerContentType.get(hostname).containsKey(contentType)) {
                totalBytesPerHostPerContentType.get(hostname).put(contentType, Integer.valueOf(bytes));
            } else {
                totalBytesPerHostPerContentType.get(hostname).put(contentType, totalBytesPerHostPerContentType.get(hostname).get(contentType) + Integer.valueOf(bytes));
            }


            // Track total TTFB for host
            if (!totalTTFBPerHost.containsKey(hostname)) {
                totalTTFBPerHost.put(hostname, Integer.valueOf(ttfb));
            } else {
                totalTTFBPerHost.put(hostname, totalTTFBPerHost.get(hostname) + Integer.valueOf(ttfb));
            }

            // Track total DNS lookup time for host
            if (!totalDNSLookupPerHost.containsKey(hostname)) {
                totalDNSLookupPerHost.put(hostname, Integer.valueOf(dns));
            } else {
                totalDNSLookupPerHost.put(hostname, totalDNSLookupPerHost.get(hostname) + Integer.valueOf(dns));
            }

            // Track total initial connection time for host
            if (!totalInitialCnxnTimePerHost.containsKey(hostname)) {
                totalInitialCnxnTimePerHost.put(hostname, Integer.valueOf(cnxn));
            } else {
                totalInitialCnxnTimePerHost.put(hostname, totalInitialCnxnTimePerHost.get(hostname) + Integer.valueOf(cnxn));
            }

            // Track total bytes for host
            if (!totalBytesPerHost.containsKey(hostname)) {
                totalBytesPerHost.put(hostname, Integer.valueOf(bytes));
            } else {
                totalBytesPerHost.put(hostname, totalBytesPerHost.get(hostname) + Integer.valueOf(bytes));
            }
            
        }

        printMap("Total # requests per host", numRequestsPerHost);
        printMap("Total # requests per host per content-type", numRequestsPerHostPerContentType);
        printMap("Total # bytes per host per content-type", totalBytesPerHostPerContentType);
        printMap("Total TTFB per host", totalTTFBPerHost);
        printMap("Total DNS lookup per host", totalDNSLookupPerHost);
        printMap("Total Initial Connection Time per host", totalInitialCnxnTimePerHost);
        printMap("Total Bytes per host", totalBytesPerHost);
    }

    private static void printMap(String title, HashMap stats) {
        System.out.println("\t" + title);

        Iterator keyItr = stats.keySet().iterator();
        while (keyItr.hasNext()) {
            Object key = keyItr.next();
            Object value = stats.get(key);

            System.out.println("\t\t" + key.toString() + ": " + value.toString());
        }

    }

}