AdminFinder.java

import java.io.BufferedReader;
import java.io.FileReader;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;

/*
  _____   _       _        _____ _       _             
 |  _  |_| |_____|_|___   |   __|_|___ _| |___ ___     
 |     | . |     | |   |  |   __| |   | . | -_|  _|    
 |__|__|___|_|_|_|_|_|_|  |__|  |_|_|_|___|___|_|      
                                [ github.com/SegoCode ]	
         Java Version;
*/


public class AdminFinder {
	public static List<String> getRobots(String site) throws Exception {
		ArrayList<String> robotsPaths = new ArrayList<String>();
		URL paths = new URL(site + "/robots.txt");
		BufferedReader br = new BufferedReader(new InputStreamReader(paths.openStream()));
		String line;
		while ((line = br.readLine()) != null) {
			robotsPaths.add(line);
		}
		br.close();
		return robotsPaths;

	}

	public static void main(String[] args) {
		try {
			System.out.println("Working Started . . . ");

			String site = args[0];

			if (site.substring(site.length() - 1).equals("/")) {
				site = site.substring(0, site.length() - 1).trim();

			}

			List<String> robot = getRobots(site);

			for (String path : robot) {
				if (path.contains("Disallow:")) {
					path = path.replaceAll("Disallow:", "").trim();
					URLTester urlt = new URLTester(site + path);
					urlt.start();
				}
			}

			BufferedReader br = new BufferedReader(new FileReader("paths.txt"));
			String line;
			while ((line = br.readLine()) != null) {
				URLTester urlt = new URLTester(site + line);
				urlt.start();
			}
			br.close();
		} catch (Exception e) {
			System.out.println("");
		}
	}

}

class URLTester extends Thread {

	private String url;

	public URLTester(String url) {
		this.url = url;
	}

	@Override
	public void run() {
		try {
			URL urlObj = new URL(this.url);
			HttpURLConnection connection = (HttpURLConnection) urlObj.openConnection();
			connection.setRequestMethod("GET");
			connection.setConnectTimeout(3000);
			connection.connect();
			int code = connection.getResponseCode();
			if (code <= 200 && code < 300) {
				System.out.println("[+] FOUND: " + this.url + " (Response code OK - " + code + ")");
			}
			if (code == 403) {
				System.out.println("[?] FOUND (Forbidden): " + this.url + " (Response code - " + code + ")");
			}
		} catch (Exception e) {
			System.out.println("[-] ERR: " + this.url + " (TIME-OUT) ");
		}
	}
}