Created
October 19, 2013 20:06
-
-
Save noahm/7060866 to your computer and use it in GitHub Desktop.
A java-based tribunal scraper. Originally written by Lecherito (originally found here: http://pastebin.com/7Sp1eJwv)
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package Tribunal; | |
import java.io.BufferedReader; | |
import java.io.File; | |
import java.io.FileOutputStream; | |
import java.io.IOException; | |
import java.io.InputStreamReader; | |
import java.net.URL; | |
import java.net.URLConnection; | |
import java.util.ArrayList; | |
import java.util.concurrent.atomic.AtomicInteger; | |
import com.mongodb.BasicDBList; | |
import com.mongodb.DBObject; | |
import com.mongodb.util.JSON; | |
import connection.MySQL; | |
public class GetCases { | |
public static final String TRIBUNAL_URL = "http://euw.leagueoflegends.com/tribunal/en/get_reform_game/"; | |
static int veryFirstCase = 3190; | |
static int firstCase = 668370+1; | |
static int lastCase = 1313652; | |
static final int GAMES_PER_CASE = 5; | |
private static final int workers = 200; | |
static File f = new File("D:\\log.ttx"); | |
static FileOutputStream fos; | |
private final static AtomicInteger counter = new AtomicInteger(firstCase); | |
static ArrayList<Thread> hilos = new ArrayList<>(); | |
public static void main(String[] args) throws IOException { | |
fos = new FileOutputStream(f); | |
for (int i=0;i<workers;i++) { | |
hilos.add(new Thread(new Dumper())); | |
hilos.get(i).start(); | |
} | |
} | |
private static String getUrlSource(String url2) throws IOException { | |
URL url = new URL(url2); | |
URLConnection spoof = url.openConnection(); | |
spoof.setRequestProperty( | |
"User-Agent", "Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0; H010818)" ); | |
BufferedReader in = new BufferedReader(new InputStreamReader(spoof.getInputStream())); | |
String strLine = ""; | |
StringBuilder sBuilder = new StringBuilder(); | |
while ((strLine = in.readLine()) != null) { | |
sBuilder.append(strLine); | |
} | |
in.close(); | |
return sBuilder.toString(); | |
} | |
static class Dumper implements Runnable { | |
@Override | |
public void run() { | |
int caso = 0; | |
while ((caso = counter.getAndIncrement()) < lastCase) { | |
String tibunalCase = "http://euw.leagueoflegends.com/tribunal/en/case/"+caso; | |
String data = ""; | |
try { | |
data = getUrlSource(tibunalCase); | |
} catch (IOException e1) { | |
try { | |
fos.write(e1.getMessage().getBytes()); | |
} catch (IOException e) { | |
e.printStackTrace(); | |
} | |
} | |
String veredicto = ""; | |
String howMany = ""; | |
String punishment = ""; | |
try { | |
int indexVer = data.indexOf("verdict-stat\""); | |
punishment = getPunishment(data, indexVer); | |
indexVer = data.indexOf("verdict-stat agreement\"", indexVer); | |
howMany = getHowMany(data, indexVer); | |
indexVer = data.indexOf("verdict-stat\"", indexVer); | |
veredicto = getVeredict(data, indexVer); | |
} catch (Exception e) { | |
try { | |
fos.write(e.getMessage().getBytes()); | |
} catch (IOException e1) { | |
e1.printStackTrace(); | |
} | |
} | |
for (int i=1;i<=GAMES_PER_CASE;i++) { | |
String url = TRIBUNAL_URL+caso+"/"+i+"/"; | |
System.out.println("Getting "+url); | |
String urlData = "\"\""; | |
try { | |
urlData = getUrlSource(url); | |
} catch (IOException e) { e.printStackTrace(); } | |
if (urlData.equalsIgnoreCase("\"\"")) { | |
System.out.println("No more games for case "+caso); | |
break; | |
} | |
DBObject jsonObject = (DBObject) JSON.parse(urlData); | |
jsonObject.put("tribunal_id", caso); | |
jsonObject.put("case_id", i); | |
jsonObject.put("veredict", veredicto); | |
jsonObject.put("agreement", howMany); | |
jsonObject.put("punishment", punishment); | |
MySQL.getInstanciaUnica().consulta(getSqlCaseFromJson(jsonObject)); | |
MySQL.getInstanciaUnica().consulta(getSqlPlayersFromJson(jsonObject)); | |
MySQL.getInstanciaUnica().consulta(getReportsFromJson(jsonObject)); | |
} | |
} | |
} | |
private String getPunishment(String data, int index) { | |
String veredicto = data.substring(index+"verdict-stat\"".length()+1,data.indexOf("<", index)); | |
return veredicto; | |
} | |
private String getHowMany(String data, int index) { | |
String veredicto = data.substring(index+"verdict-stat agreement\"".length()+1,data.indexOf("<", index)); | |
return veredicto; | |
} | |
private String getVeredict(String data, int indexVer) { | |
String veredicto = data.substring(indexVer+"verdict-stat\"".length()+1,data.indexOf("<", indexVer)); | |
return veredicto; | |
} | |
private String getSqlCaseFromJson(DBObject json) { | |
StringBuilder sBuilder = new StringBuilder("INSERT INTO tribunal_case values ("); | |
sBuilder.append(json.get("tribunal_id")+","); | |
sBuilder.append(json.get("case_id")+","); | |
sBuilder.append("\""+json.get("game_type")+"\","); | |
sBuilder.append(json.get("platform_game_id")+","); | |
sBuilder.append(json.get("allied_report_count")+","); | |
sBuilder.append(json.get("enemy_report_count")+","); | |
sBuilder.append("\""+json.get("most_common_report_reason")+"\","); | |
sBuilder.append(json.get("case_total_reports")+","); | |
sBuilder.append("\""+json.get("veredict")+"\","); | |
sBuilder.append("\""+json.get("agreement")+"\","); | |
sBuilder.append("\""+json.get("punishment")+"\""); | |
sBuilder.append(")"); | |
return sBuilder.toString(); | |
} | |
private String getSqlPlayersFromJson(DBObject json) { | |
StringBuilder sBuilder = new StringBuilder("INSERT INTO players values "); | |
BasicDBList e = (BasicDBList) json.get("players"); | |
for (int i=0; i< e.size(); i++) { | |
DBObject dbo = (DBObject) e.get(i); | |
StringBuilder sb = new StringBuilder("("); | |
sb.append(json.get("tribunal_id")+","); | |
sb.append(json.get("case_id")+","); | |
sb.append(i+1+","); | |
sb.append("\""+dbo.get("champion_name")+"\","); | |
sb.append(dbo.get("team").toString().replaceAll("Team", "")+","); | |
sb.append(dbo.get("level")+","); | |
DBObject scores = (DBObject) dbo.get("scores"); | |
sb.append(scores.get("kills")+","); | |
sb.append(scores.get("deaths")+","); | |
sb.append(scores.get("assists")+","); | |
sb.append(scores.get("nodes_captured")+","); | |
sb.append(scores.get("nodes_neutralized")+","); | |
sb.append(scores.get("player_score")+","); | |
sb.append("\""+getItemsFromJson(dbo.get("items"))+"\","); | |
sb.append("\""+dbo.get("outcome")+"\","); | |
sb.append(dbo.get("summoner_spell_1")+","); | |
sb.append(dbo.get("summoner_spell_2")+","); | |
sb.append(dbo.get("time_played")+","); | |
sb.append("\""+dbo.get("association_to_offender")+"\")"+(i==e.size()-1?"":",")); | |
sBuilder.append(sb.toString()); | |
} | |
return sBuilder.toString(); | |
} | |
private String getItemsFromJson(Object object) { | |
BasicDBList items = (BasicDBList) object; | |
StringBuilder sBuilder = new StringBuilder(); | |
for (int i=0;i<items.size();i++) { | |
DBObject dbo = (DBObject) items.get(i); | |
sBuilder.append(dbo.get("id")+(i==items.size()-1?"":",")); | |
} | |
return sBuilder.toString(); | |
} | |
private String getReportsFromJson(DBObject json) { | |
StringBuilder sBuilder = new StringBuilder("INSERT INTO reports values "); | |
BasicDBList e = (BasicDBList) json.get("reports"); | |
for (int i=0; i< e.size(); i++) { | |
DBObject dbo = (DBObject) e.get(i); | |
StringBuilder sb = new StringBuilder("("); | |
sb.append(json.get("tribunal_id")+","); | |
sb.append(json.get("case_id")+","); | |
sb.append(i+1+","); | |
sb.append(quoteString(dbo.get("offense"))); | |
sb.append(quoteString(dbo.get("comment"))); | |
sb.append("\""+dbo.get("association_to_offender")+"\")"+(i==e.size()-1?"":",")); | |
sBuilder.append(sb.toString()); | |
} | |
return sBuilder.toString(); | |
} | |
private Object quoteString(Object object) { | |
return object==null?null:"\""+object+"\","; | |
} | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment