PROTOTYPE Google Safe Browsing URL Reputation Burp Suite scanner extension

BurpExtender.java

//
//
// PROTOTYPE Google Safe Browsing URL Reputation Burp Suite scanner extension
//
// The concept is that organisations will want to use Burp Suite Enterprise edition
// to continually scan their websites to see if they include / reference any known 
// malicious URLs i.e. they have latterly breached etc.. This is a prototype of this 
// concept using an extension and the Google Safe Browsing API.
//
// You need a API key from Google, details on how to obtain one can be found here 
// https://developers.google.com/safe-browsing/v4/lookup-api
//
// Once you have an API key you need to set an environment variable called 
//     GOOGLESBAPIKEY 
// with the key and relaunch Burp.
//
// This has been tested with Burp Suite Pro to test the concept.
//
// Ollie Whitehouse - @ollieatnowhere
//
//
package burp;

import burp.api.montoya.BurpExtension;
import burp.api.montoya.MontoyaApi;
import burp.api.montoya.logging.Logging;
import burp.api.montoya.extension.Extension;
import burp.api.montoya.extension.ExtensionUnloadingHandler;

//Burp will auto-detect and load any class that extends BurpExtension.
public class BurpExtender implements BurpExtension
{
	private Logging logging;
  
	@Override
	public void initialize(MontoyaApi api)
	{
	  
		Extension extension = api.extension();
	  
		// set extension name
		api.extension().setName("Google Safe Browsing URL Reputation");

		logging = api.logging();

		// register a new extension unload handler
		extension.registerUnloadingHandler(new MyExtensionUnloadHandler());
      
		// write a message to our output stream
		logging.logToOutput("URL Reputation v0.1");
		
		String API_KEY = System.getenv("GOOGLESBAPIKEY");
		if(API_KEY == null || API_KEY.isEmpty() || API_KEY=="") {
			logging.logToError("No Google Safe Browing API Key Supplied - set via environment variable 'GOOGLESBAPIKEY' - unloading");
			logging.logToOutput("No Google Safe Browing API Key Supplied - set via environment variable 'GOOGLESBAPIKEY' - unloading");
			api.extension().unload();
		} else {
			logging.logToOutput("Google Safe Browing API Key Supplied - we are OK."); 
		}
		
		// register the scan check
		api.scanner().registerScanCheck(new MyScanCheck(api,logging));
	}
  
	// Handler
	private class MyExtensionUnloadHandler implements ExtensionUnloadingHandler {
		@Override
		public void extensionUnloaded() {
			logging.logToOutput("Extension was unloaded.");
		}
	}
}

MyScanCheck.java

//
//
// PROTOTYPE Google Safe Browsing URL Reputation Burp Suite scanner extension
//
// The concept is that organisations will want to use Burp Suite Enterprise edition
// to continually scan their websites to see if they include / reference any known 
// malicious URLs i.e. they have latterly breached etc.. This is a prototype of this 
// concept using an extension and the Google Safe Browsing API.
//
// You need a API key from Google, details on how to obtain one can be found here 
// https://developers.google.com/safe-browsing/v4/lookup-api
//
// Once you have an API key you need to set an environment variable called 
//     GOOGLESBAPIKEY 
// with the key and relaunch Burp.
//
// This has been tested with Burp Suite Pro to test the concept.
//
// Ollie Whitehouse - @ollieatnowhere
//
//

package burp;

import java.net.URI;
import java.util.*;
import java.net.http.*;
import java.net.http.HttpRequest.BodyPublisher;
import java.net.http.HttpResponse.BodyHandlers;
import java.time.Duration;
import java.time.LocalDateTime;
import static java.util.Collections.emptyList;

import burp.api.montoya.MontoyaApi;
import burp.api.montoya.http.message.HttpRequestResponse;
import burp.api.montoya.http.message.requests.HttpRequest;
import burp.api.montoya.scanner.AuditResult;
import static burp.api.montoya.scanner.AuditResult.auditResult;
import static burp.api.montoya.scanner.audit.issues.AuditIssue.auditIssue;
import static burp.api.montoya.scanner.ConsolidationAction.KEEP_BOTH;
import static burp.api.montoya.scanner.ConsolidationAction.KEEP_EXISTING;
import burp.api.montoya.scanner.ConsolidationAction;
import burp.api.montoya.scanner.ScanCheck;
import burp.api.montoya.logging.Logging;
import burp.api.montoya.scanner.audit.insertionpoint.AuditInsertionPoint;
import burp.api.montoya.scanner.audit.issues.AuditIssue;
import burp.api.montoya.scanner.audit.issues.AuditIssueConfidence;
import burp.api.montoya.scanner.audit.issues.AuditIssueSeverity;
import burp.api.montoya.sitemap.SiteMap;


class MyScanCheck implements ScanCheck
{
    private final MontoyaApi api;
    private final Logging logging;
    private boolean weRan=false;
    private LocalDateTime last = null; 
    
    MyScanCheck(MontoyaApi api, Logging logging)
    {
        this.api = api;
        this.logging = logging;
    }  
    
    //
    // Apologies for the filth here
    //
    private List<String> buildRequests(List<String> URLs) {
    	
    	int Count = 0;
    	List<String> Requests = new ArrayList<String>();
    	String Request = new String();
    	List<String> myURLs = URLs;
    	
	    for (String tempURL: myURLs) {
	    	
	    	if(Count==0) {
		    	Request = "  {\r\n"
		    			+ "    \"client\": {\r\n"
		    			+ "      \"clientId\":      \"URL Reputation Burp Suite Extension\",\r\n"
		    			+ "      \"clientVersion\": \"1.0.0\"\r\n"
		    			+ "    },\r\n"
		    			+ "    \"threatInfo\": {\r\n"
		    			+ "      \"threatTypes\":      [\"MALWARE\", \"SOCIAL_ENGINEERING\", \"UNWANTED_SOFTWARE\", \"POTENTIALLY_HARMFUL_APPLICATION\"],\r\n"
		    			+ "      \"platformTypes\":    [\"ANY_PLATFORM\"],\r\n"
		    			+ "      \"threatEntryTypes\": [\"URL\"],\r\n"
		    			+ "      \"threatEntries\": [\r\n";
	    	}
	    	
	    	
	    	// FIXME: JSON injection to resolve for defence in depth
    		Request = Request + "        {\"url\": \"" + tempURL + "\"},\r\n"; 
    		Count++;
	    		
    		if(Count==499)
    		{
    	    	Request = Request 
    	    			+ "      ]\r\n"
    	    			+ "    }\r\n"
    	    			+ "  }";
    	    	
    	    	Count=0;
    	    	Requests.add(Request);
    		}
    	}
	    	
	    if(Count<499) {
	    	Request = Request 
    			+ "      ]\r\n"
    			+ "    }\r\n"
    			+ "  }";
    		Requests.add(Request);
	    }
    	
    	return Requests;
    	
    }
    
    
    @Override
    public AuditResult activeAudit(HttpRequestResponse baseRequestResponse, AuditInsertionPoint auditInsertionPoint)
    {
    	
    	// Result
    	List<AuditIssue> auditIssueList = new ArrayList<AuditIssue>();
    	
    	//
    	// How we achieve once per scan
    	//
    	
    	// Keep us ticking over this assumes that during a scan 
    	// we will be called at least once every 120 seconds
    	// if longer than 120 seconds since last being called reset the 
    	// the counter and assume this is a new scan
    	if(last==null) last = LocalDateTime.now();
    	LocalDateTime now = LocalDateTime.now();
    	long Delta = Duration.between(last, now).toSeconds();
		
    	// Reset
    	if(weRan == true && Delta >120) {
			logging.logToOutput("Resetting lock due to delta");
			weRan = false;
			
		// If the first time we have run this scan
		} else if(weRan==false) {
			weRan=true;
			logging.logToOutput("Checking reputations for URLs in sitemap..");

			List<String> URLs = new ArrayList<String>();
			
			// Iterate through the site map and build the bits we want to check
    	  	for (HttpRequestResponse tempHRR : this.api.siteMap().requestResponses()) {
    	  		String URL = null;
    	  		int End = tempHRR.url().toString().indexOf("?");
    	  		
    	  		if (End != -1) 
    	  		{
    	  			URL = tempHRR.url().toString().substring(0, End);	
    	  		} else {
    	  			URL = tempHRR.url().toString();
    	  		}
    	  		
    	  		if(URLs.contains(URL) == false) {
    	  			URLs.add(URL);
    	  		}
    	  	}
    	  	  	  	
    	  	// HTTP Client
    	  	java.net.http.HttpClient client = HttpClient.newHttpClient();
    	  	
    	  	// Build the requests
    	  	List<String> Requests = new ArrayList<String>();
    	  	Requests = buildRequests(URLs);
    	  	logging.logToOutput("Built this number of requests " + Requests.size());
    	  	
    	  	for (String tempRequest: Requests) {

    	  		//
    	  		String API_KEY = System.getenv("GOOGLESBAPIKEY");
    	    	
    	    	try {
    	    		   	    		
    	    		// Build the POST request
    	    		java.net.http.HttpRequest request = java.net.http.HttpRequest.newBuilder()
    	    			  .uri(new URI("https://safebrowsing.googleapis.com/v4/threatMatches:find?key=" + API_KEY))
    	    			  .headers("Content-Type", "application/json")
    	    			  .POST(java.net.http.HttpRequest.BodyPublishers.ofString(tempRequest))
    	    			  .build();
    	    		    	    		
    	    		// Sent the POST request
    	    		java.net.http.HttpResponse<String> response = client.send(request, BodyHandlers.ofString());
    	    	
    	    		// Result
    	    		if(response.statusCode() != 200) {
    	    			logging.logToOutput("Recived Error:" + response.statusCode());
    	    			return null;
    	    		} else {
    	    			logging.logToOutput("Received 200: " + response.body().length());
    	    			if(response.body().length() > 3 ) {
    	    				logging.logToOutput("We got a bad reputation URL");
    	    				
    	    				AuditIssue myIssue = auditIssue(
    		                        "URL Reputation",
    		                        "Google has flagged a URL with bad reputation " + response.body().toString(),
    		                        null,
    		                        baseRequestResponse.request().url(),
    		                        AuditIssueSeverity.HIGH,
    		                        AuditIssueConfidence.CERTAIN,
    		                        null,
    		                        null,
    		                        AuditIssueSeverity.HIGH,
    		                        baseRequestResponse
    		                	);
    	    				
    	    				auditIssueList.add(myIssue);
    	    		                	
    	    			} else {
    	    				
    	    			}
    	    		}
    	    		
    	    	} catch(Exception e) {
    	    		logging.logToError("Exception generated when checking Google safe browsing");
    	    	}
    	  	}
	    	
	    }
		
    	// 
		last = LocalDateTime.now();
    	
		return auditResult(auditIssueList);
    }
    
    @Override
    public AuditResult passiveAudit(HttpRequestResponse baseRequestResponse)
    {
    	List<AuditIssue> auditIssueList = emptyList(); 
    	return auditResult(auditIssueList);
    }
    
    @Override
    public ConsolidationAction consolidateIssues(AuditIssue newIssue, AuditIssue existingIssue)
    {
    	return existingIssue.name().equals(newIssue.name()) ? KEEP_EXISTING : KEEP_BOTH;
    }

}