Skip to content

Instantly share code, notes, and snippets.

@noahm
Created October 19, 2013 20:06
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save noahm/7060866 to your computer and use it in GitHub Desktop.
Save noahm/7060866 to your computer and use it in GitHub Desktop.
A java-based tribunal scraper. Originally written by Lecherito (originally found here: http://pastebin.com/7Sp1eJwv)
package Tribunal;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.concurrent.atomic.AtomicInteger;
import com.mongodb.BasicDBList;
import com.mongodb.DBObject;
import com.mongodb.util.JSON;
import connection.MySQL;
public class GetCases {
public static final String TRIBUNAL_URL = "http://euw.leagueoflegends.com/tribunal/en/get_reform_game/";
static int veryFirstCase = 3190;
static int firstCase = 668370+1;
static int lastCase = 1313652;
static final int GAMES_PER_CASE = 5;
private static final int workers = 200;
static File f = new File("D:\\log.ttx");
static FileOutputStream fos;
private final static AtomicInteger counter = new AtomicInteger(firstCase);
static ArrayList<Thread> hilos = new ArrayList<>();
public static void main(String[] args) throws IOException {
fos = new FileOutputStream(f);
for (int i=0;i<workers;i++) {
hilos.add(new Thread(new Dumper()));
hilos.get(i).start();
}
}
private static String getUrlSource(String url2) throws IOException {
URL url = new URL(url2);
URLConnection spoof = url.openConnection();
spoof.setRequestProperty(
"User-Agent", "Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0; H010818)" );
BufferedReader in = new BufferedReader(new InputStreamReader(spoof.getInputStream()));
String strLine = "";
StringBuilder sBuilder = new StringBuilder();
while ((strLine = in.readLine()) != null) {
sBuilder.append(strLine);
}
in.close();
return sBuilder.toString();
}
static class Dumper implements Runnable {
@Override
public void run() {
int caso = 0;
while ((caso = counter.getAndIncrement()) < lastCase) {
String tibunalCase = "http://euw.leagueoflegends.com/tribunal/en/case/"+caso;
String data = "";
try {
data = getUrlSource(tibunalCase);
} catch (IOException e1) {
try {
fos.write(e1.getMessage().getBytes());
} catch (IOException e) {
e.printStackTrace();
}
}
String veredicto = "";
String howMany = "";
String punishment = "";
try {
int indexVer = data.indexOf("verdict-stat\"");
punishment = getPunishment(data, indexVer);
indexVer = data.indexOf("verdict-stat agreement\"", indexVer);
howMany = getHowMany(data, indexVer);
indexVer = data.indexOf("verdict-stat\"", indexVer);
veredicto = getVeredict(data, indexVer);
} catch (Exception e) {
try {
fos.write(e.getMessage().getBytes());
} catch (IOException e1) {
e1.printStackTrace();
}
}
for (int i=1;i<=GAMES_PER_CASE;i++) {
String url = TRIBUNAL_URL+caso+"/"+i+"/";
System.out.println("Getting "+url);
String urlData = "\"\"";
try {
urlData = getUrlSource(url);
} catch (IOException e) { e.printStackTrace(); }
if (urlData.equalsIgnoreCase("\"\"")) {
System.out.println("No more games for case "+caso);
break;
}
DBObject jsonObject = (DBObject) JSON.parse(urlData);
jsonObject.put("tribunal_id", caso);
jsonObject.put("case_id", i);
jsonObject.put("veredict", veredicto);
jsonObject.put("agreement", howMany);
jsonObject.put("punishment", punishment);
MySQL.getInstanciaUnica().consulta(getSqlCaseFromJson(jsonObject));
MySQL.getInstanciaUnica().consulta(getSqlPlayersFromJson(jsonObject));
MySQL.getInstanciaUnica().consulta(getReportsFromJson(jsonObject));
}
}
}
private String getPunishment(String data, int index) {
String veredicto = data.substring(index+"verdict-stat\"".length()+1,data.indexOf("<", index));
return veredicto;
}
private String getHowMany(String data, int index) {
String veredicto = data.substring(index+"verdict-stat agreement\"".length()+1,data.indexOf("<", index));
return veredicto;
}
private String getVeredict(String data, int indexVer) {
String veredicto = data.substring(indexVer+"verdict-stat\"".length()+1,data.indexOf("<", indexVer));
return veredicto;
}
private String getSqlCaseFromJson(DBObject json) {
StringBuilder sBuilder = new StringBuilder("INSERT INTO tribunal_case values (");
sBuilder.append(json.get("tribunal_id")+",");
sBuilder.append(json.get("case_id")+",");
sBuilder.append("\""+json.get("game_type")+"\",");
sBuilder.append(json.get("platform_game_id")+",");
sBuilder.append(json.get("allied_report_count")+",");
sBuilder.append(json.get("enemy_report_count")+",");
sBuilder.append("\""+json.get("most_common_report_reason")+"\",");
sBuilder.append(json.get("case_total_reports")+",");
sBuilder.append("\""+json.get("veredict")+"\",");
sBuilder.append("\""+json.get("agreement")+"\",");
sBuilder.append("\""+json.get("punishment")+"\"");
sBuilder.append(")");
return sBuilder.toString();
}
private String getSqlPlayersFromJson(DBObject json) {
StringBuilder sBuilder = new StringBuilder("INSERT INTO players values ");
BasicDBList e = (BasicDBList) json.get("players");
for (int i=0; i< e.size(); i++) {
DBObject dbo = (DBObject) e.get(i);
StringBuilder sb = new StringBuilder("(");
sb.append(json.get("tribunal_id")+",");
sb.append(json.get("case_id")+",");
sb.append(i+1+",");
sb.append("\""+dbo.get("champion_name")+"\",");
sb.append(dbo.get("team").toString().replaceAll("Team", "")+",");
sb.append(dbo.get("level")+",");
DBObject scores = (DBObject) dbo.get("scores");
sb.append(scores.get("kills")+",");
sb.append(scores.get("deaths")+",");
sb.append(scores.get("assists")+",");
sb.append(scores.get("nodes_captured")+",");
sb.append(scores.get("nodes_neutralized")+",");
sb.append(scores.get("player_score")+",");
sb.append("\""+getItemsFromJson(dbo.get("items"))+"\",");
sb.append("\""+dbo.get("outcome")+"\",");
sb.append(dbo.get("summoner_spell_1")+",");
sb.append(dbo.get("summoner_spell_2")+",");
sb.append(dbo.get("time_played")+",");
sb.append("\""+dbo.get("association_to_offender")+"\")"+(i==e.size()-1?"":","));
sBuilder.append(sb.toString());
}
return sBuilder.toString();
}
private String getItemsFromJson(Object object) {
BasicDBList items = (BasicDBList) object;
StringBuilder sBuilder = new StringBuilder();
for (int i=0;i<items.size();i++) {
DBObject dbo = (DBObject) items.get(i);
sBuilder.append(dbo.get("id")+(i==items.size()-1?"":","));
}
return sBuilder.toString();
}
private String getReportsFromJson(DBObject json) {
StringBuilder sBuilder = new StringBuilder("INSERT INTO reports values ");
BasicDBList e = (BasicDBList) json.get("reports");
for (int i=0; i< e.size(); i++) {
DBObject dbo = (DBObject) e.get(i);
StringBuilder sb = new StringBuilder("(");
sb.append(json.get("tribunal_id")+",");
sb.append(json.get("case_id")+",");
sb.append(i+1+",");
sb.append(quoteString(dbo.get("offense")));
sb.append(quoteString(dbo.get("comment")));
sb.append("\""+dbo.get("association_to_offender")+"\")"+(i==e.size()-1?"":","));
sBuilder.append(sb.toString());
}
return sBuilder.toString();
}
private Object quoteString(Object object) {
return object==null?null:"\""+object+"\",";
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment