Last active
January 26, 2017 20:08
-
-
Save FFrost/7c85bb9ff12c1b9110a0 to your computer and use it in GitHub Desktop.
Python script that checks if Steam community URLs are available
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import requests, urllib, sys, os.path, time, threading | |
from lxml import html | |
from string import ascii_lowercase | |
from itertools import product | |
debugmode = False | |
version = "1.2" | |
print("Steam URL Checker version %s") % version | |
wordsite = "http://wordfind.com/" | |
wordsup = True | |
steamurl = "http://steamcommunity.com/id/" | |
# check sites (if they're not up, we can't do anything) | |
sitecode = urllib.urlopen(wordsite).getcode() | |
if (sitecode != 200): | |
print("Error [%d]! Word site is down, can't search for real words!") % sitecode | |
wordsup = False | |
sitecode = urllib.urlopen(steamurl).getcode() | |
if (sitecode != 200): | |
print("Error [%d]! Steam site is down, aborting!") % sitecode | |
sys.exit(0) | |
# check steam profile for word (/id/word) | |
def checkSteam(word): | |
checksteamurl = steamurl + word | |
while True: | |
global url | |
try: | |
url = requests.get(checksteamurl) | |
except requests.ConnectionError: | |
print("Failed to connect to Steam site, retrying!") | |
time.sleep(2) | |
continue | |
tree = html.fromstring(url.text) | |
msg = tree.xpath("//div/h3/text()") # <div id='message'> ... <h3>profile</h3> ... </div> | |
if (msg): # if ^ found anything (this means the profile does not exist, will be empty if the profile exists) | |
if (msg[0] == "The specified profile could not be found."): | |
return True | |
else: | |
return False | |
return False | |
# search word site for words to use | |
def search(num): | |
global site | |
url = wordsite + num + "-letter-words/" | |
while True: | |
try: | |
site = requests.get(url) | |
except requests.ConnectionError: | |
print("Failed to connect to word site, retrying!") | |
time.sleep(2) | |
continue | |
tree = html.fromstring(site.text) | |
words = tree.xpath("//li[@class='defLink']/a[text()]/@href") # gets words in "/word/<word>/" form | |
if not words: # no words are that length | |
return False, None | |
for i, word in enumerate(words): | |
words[i] = word[6:len(word) - 1] # get rid of the beginning "/word/" and the last "/" | |
return True, words | |
return False, None | |
# generate string of letters num length, thanks to stackoverflow | |
def searchAll(num): | |
num = int(num, 10) | |
words = [] | |
for combo in product(ascii_lowercase, repeat=num): | |
str = ''.join(combo) | |
words.append(str) | |
return True, words | |
# split array of words into num chunks for num threads to loop concurrently | |
def split(array, num): | |
size = len(array) | |
splitsize = size / num | |
if (size % splitsize != 0): | |
splitsize += 1 | |
ret = [] | |
for i in range(0, num): | |
ret.append(array[i*splitsize:i*splitsize + splitsize]) | |
return ret | |
# do the looping and profile checking here | |
threadedarray = [] | |
def analysis(length, urls, threadnum): | |
numsaved = 0 | |
wordarray = [] | |
for word in urls: | |
add = "" | |
if (threadnum > 0 and debugmode): | |
add = "Thread #%d: " % threadnum | |
if (checkSteam(word)): | |
print("%s[Available] %s") % (add, steamurl + word) | |
numsaved += 1 | |
wordarray.append(word) | |
threadedarray.append(word) | |
elif (bprinttaken): | |
print("%s[Taken] %s") % (add, word) | |
if (threadnum < 0): | |
print("Finished searching!") | |
else: | |
print("Thread #%d finished searching!") % threadnum | |
return | |
# options | |
# debug mode | |
inp = raw_input("Run in debug mode? (y/n): ") | |
if (inp.lower() == "y"): | |
debugmode = True | |
print("Running in debug mode!") | |
else: | |
print("Running normally!") | |
# print [taken] or just [available] | |
bprinttaken = True | |
inp = raw_input("Only print avilable URLs? (y/n): ") | |
if (inp.lower() == "y"): | |
bprinttaken = False | |
print("Only printing available URLs!") | |
else: | |
print("Printing avilable and unavailable URLs!") | |
# :) | |
bmultithreaded = False | |
inp = raw_input("Use multithreading? (y/n): ") | |
if (inp.lower() == "y"): | |
bmultithreaded = True | |
print("Multithreading enabled!") | |
else: | |
print("Multithreading disabled!") | |
# # of threads | |
ithreads = 0 | |
if (bmultithreaded): | |
while True: | |
inp = raw_input("How many threads (at least 2): ") | |
if (not inp.isdigit()): | |
print("Please enter a number!") | |
continue | |
ithreads = int(inp, 10) | |
print("Using %d threads!") % ithreads | |
break | |
# search all/words | |
bsearchall = False | |
while True: | |
searchvar = raw_input("Enter 'all' to search all possible URLs (ex. aaa-zzz) or 'words' to search real words only: ") | |
if (not searchvar.lower() == "all" and searchvar.lower() != "words"): | |
print("Please enter 'all' or 'words'!") | |
continue | |
elif (searchvar.lower() == "all"): | |
bsearchall = True | |
print("Using all possible URLs!") | |
break | |
elif (searchvar.lower() == "words"): | |
if (not wordsup): | |
print("[Error] Word site is down, can't search for real words!") | |
continue | |
print("Using only real words!") | |
bsearchall = False | |
break | |
# word length and elapsed time | |
flength = 0 | |
elptime = 0 | |
# start the program | |
while True: | |
urllength = "7" | |
if (bsearchall): | |
urllength = "32" | |
length = raw_input("Enter how long the URL should be (3-" + urllength + "): ") | |
if(not length.isdigit()): | |
print("Please enter a number!") | |
continue | |
ilength = int(length, 10) | |
if (ilength <= 2 or ilength > int(urllength, 10)): | |
print("Please enter a number between 3 and %s!") % urllength | |
continue | |
flength = ilength | |
print("Finding URLs of length %s...") % length | |
# do the searching | |
exists, urls = False, None | |
if (bsearchall): | |
exists, urls = searchAll(length) | |
else: | |
exists, urls = search(length) | |
if (urls): | |
print("Found %d words!") % len(urls) | |
if (bmultithreaded): | |
print("Estimated time: %dm %ds (for %d threads)") % (len(urls) * 0.5 / 60 / ithreads, ((len(urls) * 0.5) / ithreads) % 60 , ithreads) | |
else: | |
print("Estimated time: %dm") % ((len(urls) * 0.5) / 60) | |
inp = raw_input("Press enter to start... ") | |
elptime = time.time() | |
if (bmultithreaded): | |
print("Setting up threads...") | |
threads = [] | |
chunks = split(urls, ithreads) | |
threadnum = 0 | |
for array in chunks: | |
threadnum += 1 | |
t = threading.Thread(target=analysis, args=(length, array, threadnum,)) | |
threads.append(t) | |
t.start() | |
print("Thread #%d starting!") % threadnum | |
for thread in threads: | |
thread.join() # process (loop words, check profiles, etc) before saving/finishing program | |
break # break out of while loop | |
else: | |
print("Searching steam...") | |
analysis(length, urls, -1) # single-threaded | |
else: | |
print("[Error] Couldn't find any words of length %s. Please try again.") % length | |
# file writing/finishing up the program | |
endtime = time.time() | |
tottime = (endtime - elptime) / 60 # minutes | |
tottime_s = (endtime - elptime) % 60 # seconds | |
flength = str(flength) | |
if (not threadedarray): | |
print("No available URLs of length %s!") % flength | |
else: | |
print("Finished searching! Took %dm %ds!") % (tottime, tottime_s) | |
filename = flength + "words.txt" | |
if (bsearchall): | |
filename = flength + "all.txt" | |
threadedarray = sorted(threadedarray) | |
print("Saving %d available URLs to %s...") % (len(threadedarray), filename) | |
f = open(filename, "w") | |
fstr = "%s\t Length: %s\t Number of URLs: %d\t Elapsed time: %dm %ds\n" % ("All" if bsearchall else "Words", flength, len(threadedarray), tottime, tottime_s) | |
f.write(fstr) | |
for w in threadedarray: | |
f.write(w + "\n") | |
f.close() | |
print("Saved!") | |
inp = raw_input("Press enter to quit. ") # keep the window open until the user wants to close it | |
sys.exit(0) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
my code