Skip to content

Instantly share code, notes, and snippets.

@FFrost
Last active January 26, 2017 20:08
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save FFrost/7c85bb9ff12c1b9110a0 to your computer and use it in GitHub Desktop.
Save FFrost/7c85bb9ff12c1b9110a0 to your computer and use it in GitHub Desktop.
Python script that checks if Steam community URLs are available
import requests, urllib, sys, os.path, time, threading
from lxml import html
from string import ascii_lowercase
from itertools import product
debugmode = False
version = "1.2"
print("Steam URL Checker version %s") % version
wordsite = "http://wordfind.com/"
wordsup = True
steamurl = "http://steamcommunity.com/id/"
# check sites (if they're not up, we can't do anything)
sitecode = urllib.urlopen(wordsite).getcode()
if (sitecode != 200):
print("Error [%d]! Word site is down, can't search for real words!") % sitecode
wordsup = False
sitecode = urllib.urlopen(steamurl).getcode()
if (sitecode != 200):
print("Error [%d]! Steam site is down, aborting!") % sitecode
sys.exit(0)
# check steam profile for word (/id/word)
def checkSteam(word):
checksteamurl = steamurl + word
while True:
global url
try:
url = requests.get(checksteamurl)
except requests.ConnectionError:
print("Failed to connect to Steam site, retrying!")
time.sleep(2)
continue
tree = html.fromstring(url.text)
msg = tree.xpath("//div/h3/text()") # <div id='message'> ... <h3>profile</h3> ... </div>
if (msg): # if ^ found anything (this means the profile does not exist, will be empty if the profile exists)
if (msg[0] == "The specified profile could not be found."):
return True
else:
return False
return False
# search word site for words to use
def search(num):
global site
url = wordsite + num + "-letter-words/"
while True:
try:
site = requests.get(url)
except requests.ConnectionError:
print("Failed to connect to word site, retrying!")
time.sleep(2)
continue
tree = html.fromstring(site.text)
words = tree.xpath("//li[@class='defLink']/a[text()]/@href") # gets words in "/word/<word>/" form
if not words: # no words are that length
return False, None
for i, word in enumerate(words):
words[i] = word[6:len(word) - 1] # get rid of the beginning "/word/" and the last "/"
return True, words
return False, None
# generate string of letters num length, thanks to stackoverflow
def searchAll(num):
num = int(num, 10)
words = []
for combo in product(ascii_lowercase, repeat=num):
str = ''.join(combo)
words.append(str)
return True, words
# split array of words into num chunks for num threads to loop concurrently
def split(array, num):
size = len(array)
splitsize = size / num
if (size % splitsize != 0):
splitsize += 1
ret = []
for i in range(0, num):
ret.append(array[i*splitsize:i*splitsize + splitsize])
return ret
# do the looping and profile checking here
threadedarray = []
def analysis(length, urls, threadnum):
numsaved = 0
wordarray = []
for word in urls:
add = ""
if (threadnum > 0 and debugmode):
add = "Thread #%d: " % threadnum
if (checkSteam(word)):
print("%s[Available] %s") % (add, steamurl + word)
numsaved += 1
wordarray.append(word)
threadedarray.append(word)
elif (bprinttaken):
print("%s[Taken] %s") % (add, word)
if (threadnum < 0):
print("Finished searching!")
else:
print("Thread #%d finished searching!") % threadnum
return
# options
# debug mode
inp = raw_input("Run in debug mode? (y/n): ")
if (inp.lower() == "y"):
debugmode = True
print("Running in debug mode!")
else:
print("Running normally!")
# print [taken] or just [available]
bprinttaken = True
inp = raw_input("Only print avilable URLs? (y/n): ")
if (inp.lower() == "y"):
bprinttaken = False
print("Only printing available URLs!")
else:
print("Printing avilable and unavailable URLs!")
# :)
bmultithreaded = False
inp = raw_input("Use multithreading? (y/n): ")
if (inp.lower() == "y"):
bmultithreaded = True
print("Multithreading enabled!")
else:
print("Multithreading disabled!")
# # of threads
ithreads = 0
if (bmultithreaded):
while True:
inp = raw_input("How many threads (at least 2): ")
if (not inp.isdigit()):
print("Please enter a number!")
continue
ithreads = int(inp, 10)
print("Using %d threads!") % ithreads
break
# search all/words
bsearchall = False
while True:
searchvar = raw_input("Enter 'all' to search all possible URLs (ex. aaa-zzz) or 'words' to search real words only: ")
if (not searchvar.lower() == "all" and searchvar.lower() != "words"):
print("Please enter 'all' or 'words'!")
continue
elif (searchvar.lower() == "all"):
bsearchall = True
print("Using all possible URLs!")
break
elif (searchvar.lower() == "words"):
if (not wordsup):
print("[Error] Word site is down, can't search for real words!")
continue
print("Using only real words!")
bsearchall = False
break
# word length and elapsed time
flength = 0
elptime = 0
# start the program
while True:
urllength = "7"
if (bsearchall):
urllength = "32"
length = raw_input("Enter how long the URL should be (3-" + urllength + "): ")
if(not length.isdigit()):
print("Please enter a number!")
continue
ilength = int(length, 10)
if (ilength <= 2 or ilength > int(urllength, 10)):
print("Please enter a number between 3 and %s!") % urllength
continue
flength = ilength
print("Finding URLs of length %s...") % length
# do the searching
exists, urls = False, None
if (bsearchall):
exists, urls = searchAll(length)
else:
exists, urls = search(length)
if (urls):
print("Found %d words!") % len(urls)
if (bmultithreaded):
print("Estimated time: %dm %ds (for %d threads)") % (len(urls) * 0.5 / 60 / ithreads, ((len(urls) * 0.5) / ithreads) % 60 , ithreads)
else:
print("Estimated time: %dm") % ((len(urls) * 0.5) / 60)
inp = raw_input("Press enter to start... ")
elptime = time.time()
if (bmultithreaded):
print("Setting up threads...")
threads = []
chunks = split(urls, ithreads)
threadnum = 0
for array in chunks:
threadnum += 1
t = threading.Thread(target=analysis, args=(length, array, threadnum,))
threads.append(t)
t.start()
print("Thread #%d starting!") % threadnum
for thread in threads:
thread.join() # process (loop words, check profiles, etc) before saving/finishing program
break # break out of while loop
else:
print("Searching steam...")
analysis(length, urls, -1) # single-threaded
else:
print("[Error] Couldn't find any words of length %s. Please try again.") % length
# file writing/finishing up the program
endtime = time.time()
tottime = (endtime - elptime) / 60 # minutes
tottime_s = (endtime - elptime) % 60 # seconds
flength = str(flength)
if (not threadedarray):
print("No available URLs of length %s!") % flength
else:
print("Finished searching! Took %dm %ds!") % (tottime, tottime_s)
filename = flength + "words.txt"
if (bsearchall):
filename = flength + "all.txt"
threadedarray = sorted(threadedarray)
print("Saving %d available URLs to %s...") % (len(threadedarray), filename)
f = open(filename, "w")
fstr = "%s\t Length: %s\t Number of URLs: %d\t Elapsed time: %dm %ds\n" % ("All" if bsearchall else "Words", flength, len(threadedarray), tottime, tottime_s)
f.write(fstr)
for w in threadedarray:
f.write(w + "\n")
f.close()
print("Saved!")
inp = raw_input("Press enter to quit. ") # keep the window open until the user wants to close it
sys.exit(0)
@angelfor3v3r
Copy link

my code

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment