Skip to content

Instantly share code, notes, and snippets.

@lazear
Created November 29, 2022 22:34
Show Gist options
  • Save lazear/b43b0d47f0851792c363d21cf50cd132 to your computer and use it in GitHub Desktop.
Save lazear/b43b0d47f0851792c363d21cf50cd132 to your computer and use it in GitHub Desktop.
import requests
from multiprocessing import Pool
TOKEN = "eyJhbGciOiJSUzI1NiJ9.<your JWT here>"
def download(f):
print(f"downloading {f['name']}...")
try:
resp = requests.get(f["href"])
with open(f["name"], 'wb') as file:
file.write(resp.content)
except Exception as ex:
print(f"exception while downloading {f['name']}: {ex}")
if __name__ == '__main__':
files = 'https://www.ebi.ac.uk/pride/private/ws/archive/v2/projects/<YOUR PXD NUMBER HERE>/files'
fs = []
while True:
f = requests.get(files, headers={'Authorization':f'Bearer {TOKEN}'}).json()
if '_embedded' not in f:
break
files = f['_embedded']['files']
if len(files) == 0:
break
for file in files:
fs.append({
'name': file['fileName'],
'href': file['_links']['download']['href']
})
nxt = f['_links']['next']['href']
files = nxt
with Pool(processes=8) as pool:
for x in pool.imap_unordered(download, filter(lambda x: x['name'].endswith('.raw'), fs)):
pass
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment