Skip to content

Instantly share code, notes, and snippets.

@neuromusic

neuromusic/crcns download.ipynb

Created June 30, 2016 06:08
Show Gist options
  • Save neuromusic/ef7672f2142efcb22adb015bac20d029 to your computer and use it in GitHub Desktop.
Save neuromusic/ef7672f2142efcb22adb015bac20d029 to your computer and use it in GitHub Desktop.
Download ZIP
downloading from crcns
Display the source blob
Display the rendered blob
Raw
crcns download.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"import requests"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false,
"scrolled": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{'content-length': '3340', 'x-powered-by': 'PHP/5.3.3', 'set-cookie': 'crcns_nersc_download=id6582key607477388', 'accept-ranges': 'bytes', 'server': 'Apache/2.2.3 (CentOS)', 'connection': 'close', 'date': 'Thu, 30 Jun 2016 05:15:40 GMT', 'content-type': 'text/html; charset=UTF-8'}\n",
"\n",
"<html>\n",
"<head>\n",
"<title>CRCNS.org data download</title>\n",
"</head>\n",
"<body>\n",
"<h2>CRCNS.org data download</h2>\n",
"<p>Logged in as jkiggins (Justin Kiggins). <a href=\"/project/crcns/download/index.php?cmd=logout\">logout</a>.<br />\n",
"Contents of alm-1/datafiles/data_structure_files:</p>\n",
"<a href=\"/project/crcns/download/alm-1/datafiles\">..</a><br />\n",
"<a href=\"/project/crcns/download/alm-1/datafiles/data_structure_files/data_structure_ANM210861.tar.gz\">data_structure_ANM210861.tar.gz</a> (568625369)<br />\n",
"<a href=\"/project/crcns/download/alm-1/datafiles/data_structure_files/data_structure_ANM210862.tar.gz\">data_structure_ANM210862.tar.gz</a> (423681526)<br />\n",
"<a href=\"/project/crcns/download/alm-1/datafiles/data_structure_files/data_structure_ANM210863.tar.gz\">data_structure_ANM210863.tar.gz</a> (478127420)<br />\n",
"<a href=\"/project/crcns/download/alm-1/datafiles/data_structure_files/data_structure_ANM214427.tar.gz\">data_structure_ANM214427.tar.gz</a> (713036422)<br />\n",
"<a href=\"/project/crcns/download/alm-1/datafiles/data_structure_files/data_structure_ANM214429.tar.gz\">data_structure_ANM214429.tar.gz</a> (1064032176)<br />\n",
"<a href=\"/project/crcns/download/alm-1/datafiles/data_structure_files/data_structure_ANM214430.tar.gz\">data_structure_ANM214430.tar.gz</a> (607337575)<br />\n",
"<a href=\"/project/crcns/download/alm-1/datafiles/data_structure_files/data_structure_ANM218453.tar.gz\">data_structure_ANM218453.tar.gz</a> (651067621)<br />\n",
"<a href=\"/project/crcns/download/alm-1/datafiles/data_structure_files/data_structure_ANM218457.tar.gz\">data_structure_ANM218457.tar.gz</a> (829973651)<br />\n",
"<a href=\"/project/crcns/download/alm-1/datafiles/data_structure_files/data_structure_ANM218693.tar.gz\">data_structure_ANM218693.tar.gz</a> (415317775)<br />\n",
"<a href=\"/project/crcns/download/alm-1/datafiles/data_structure_files/data_structure_ANM219030.tar.gz\">data_structure_ANM219030.tar.gz</a> (716106206)<br />\n",
"<a href=\"/project/crcns/download/alm-1/datafiles/data_structure_files/data_structure_ANM219031.tar.gz\">data_structure_ANM219031.tar.gz</a> (663861496)<br />\n",
"<a href=\"/project/crcns/download/alm-1/datafiles/data_structure_files/data_structure_ANM219033.tar.gz\">data_structure_ANM219033.tar.gz</a> (828558913)<br />\n",
"<a href=\"/project/crcns/download/alm-1/datafiles/data_structure_files/data_structure_ANM219036.tar.gz\">data_structure_ANM219036.tar.gz</a> (751800829)<br />\n",
"<a href=\"/project/crcns/download/alm-1/datafiles/data_structure_files/data_structure_ANM219037.tar.gz\">data_structure_ANM219037.tar.gz</a> (926004948)<br />\n",
"<a href=\"/project/crcns/download/alm-1/datafiles/data_structure_files/data_structure_ANM219038.tar.gz\">data_structure_ANM219038.tar.gz</a> (1040644533)<br />\n",
"<a href=\"/project/crcns/download/alm-1/datafiles/data_structure_files/data_structure_ANM219247.tar.gz\">data_structure_ANM219247.tar.gz</a> (704992587)<br />\n",
"<a href=\"/project/crcns/download/alm-1/datafiles/data_structure_files/data_structure_ANM219248.tar.gz\">data_structure_ANM219248.tar.gz</a> (1210463535)<br />\n",
"<a href=\"/project/crcns/download/alm-1/datafiles/data_structure_files/data_structure_ANM219253.tar.gz\">data_structure_ANM219253.tar.gz</a> (743796292)<br />\n",
"<a href=\"/project/crcns/download/alm-1/datafiles/data_structure_files/data_structure_ANM221977.tar.gz\">data_structure_ANM221977.tar.gz</a> (392405930)<br />\n",
"\n",
"</body>\n",
"</html> \n",
"\n"
]
}
],
"source": [
"crcns_username = ''\n",
"crcns_password = ''\n",
"\n",
"URL = 'https://portal.nersc.gov/project/crcns/download/index.php'\n",
"\n",
"path = 'alm-1/datafiles/data_structure_files'\n",
"\n",
"login_data = dict(\n",
" username=crcns_username,\n",
" password=crcns_password,\n",
" fn=path,\n",
" submit='Login' \n",
" )\n",
"\n",
"with requests.Session() as s:\n",
" r = s.post(URL,data=login_data)\n",
" print r.headers\n",
" print r.text"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"datafile = 'alm-1/datafiles/data_structure_files/data_structure_ANM218457.tar.gz'\n",
"login_data.update(fn=datafile)\n",
"\n",
"with requests.Session() as s:\n",
" local_filename = login_data['fn'].split('/')[-1]\n",
" r = s.post(URL,data=login_data,stream=True)\n",
" print r.headers\n",
" with open(local_filename, 'wb') as f:\n",
" for chunk in r.iter_content(chunk_size=1024):\n",
" if chunk:\n",
" f.write(chunk)\n",
" print local_filename"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 2",
"language": "python",
"name": "python2"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.11"
}
},
"nbformat": 4,
"nbformat_minor": 0
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment