Skip to content

Instantly share code, notes, and snippets.

@underr
Created February 2, 2016 03:03
Show Gist options
  • Save underr/8a349404dc5957649223 to your computer and use it in GitHub Desktop.
Save underr/8a349404dc5957649223 to your computer and use it in GitHub Desktop.
import requests, json, time, os
subject = 'trains'
r = requests.get('http://8ch.net/b/catalog.json')
j = json.loads(r.text)
def download(url, name, thread):
filepath = './' + str(thread) + '/' + name
if not os.path.exists(filepath):
print('Get got: ' + name)
r = requests.get(url, stream=True)
with open(filepath, 'wb') as f:
for chunk in r.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
f.flush()
def get_got():
for t in j:
threads = t['threads']
for thread in threads:
sub = thread['sub']
if sub == subject:
num = thread['no']
dirr = './' + str(num)
if sub:
if not os.path.exists(dirr):
os.makedirs(dirr)
if sub:
r2 = requests.get('http://8ch.net/b/res/' + str(num) + '.json')
j2 = json.loads(r2.text)
posts = j2['posts']
def gk(key, value):
return value.get(key, '')
for post in posts:
if gk('filename', post):
or_filename = gk('filename', post)
filename = gk('tim', post) + gk('ext', post)
download('http://8ch.net/b/src/' + filename, or_filename + gk('ext', post), num)
if gk('extra_files', post):
for extra_file in gk('extra_files', post):
orf = gk('filename', extra_file)
fin = gk('tim', extra_file) + gk('ext', extra_file)
download('http://8ch.net/b/src/' + fin, orf + gk('ext', extra_file), num)
print('Dormindo...')
time.sleep(5 * 60)
while True:
get_got()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment