Google Cloud Storage upload and download example

gc_storage_example.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Google Cloud Storage Example**\n",
    "\n",
    "For more information on the capabilities of Google Cloud Storage Python API, see:\n",
    "\n",
    "- [https://googlecloudplatform.github.io/google-cloud-python/stable/storage-buckets.html](https://googlecloudplatform.github.io/google-cloud-python/stable/storage-buckets.html)\n",
    "- [https://googlecloudplatform.github.io/google-cloud-python/stable/storage-blobs.html](https://googlecloudplatform.github.io/google-cloud-python/stable/storage-blobs.html)\n",
    "- [https://cloud.google.com/docs/authentication/production#obtaining_and_providing_service_account_credentials_manually](https://cloud.google.com/docs/authentication/production#obtaining_and_providing_service_account_credentials_manually)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# imports required to make this example work\n",
    "import pandas as pd\n",
    "import sys\n",
    "import pandas as pd\n",
    "import os\n",
    "from google.cloud import storage\n",
    "from google.auth.exceptions import DefaultCredentialsError\n",
    "from google_auth_oauthlib import flow\n",
    "\n",
    "if sys.version_info[0] < 3: \n",
    "    from StringIO import StringIO\n",
    "else:\n",
    "    from io import StringIO"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# due to the fall through nature of this authentication & authorization code, it can take up to 20 seconds to run\n",
    "\n",
    "# setup variables for this cell and subsequent cells\n",
    "project = 'my-gcp-project'\n",
    "launch_browser = True # set to False if running via commandline environment only\n",
    "\n",
    "try:\n",
    "    client = storage.Client(project = project)\n",
    "    print \"Loaded credentials from environment\"\n",
    "except DefaultCredentialsError:\n",
    "    # that failed, so provide your own credentials file\n",
    "    try:\n",
    "        client = storage.Client.from_service_account_json(\n",
    "            os.path.join(os.path.expanduser('~'),\n",
    "                         'Downloads',\n",
    "                         'service_account_key.json'))\n",
    "        print \"Loaded credentials from provided file\"\n",
    "    except:\n",
    "        # further errors, just fall back to OAuth\n",
    "        appflow = flow.InstalledAppFlow.from_client_secrets_file(\n",
    "            # you must either have an existing client identifier, or create your own. \n",
    "            os.path.join(os.path.expanduser('~'),\n",
    "                         'Downloads',\n",
    "                         'oauth2_client_id.json'),\n",
    "            # see https://developers.google.com/identity/protocols/googlescopes\n",
    "            scopes=['https://www.googleapis.com/auth/devstorage.read_write'])\n",
    "\n",
    "        if launch_browser:\n",
    "            appflow.run_local_server()\n",
    "        else:\n",
    "            appflow.run_console()\n",
    "            \n",
    "        client = storage.Client(project=project, credentials=appflow.credentials)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# read a Pandas dataframe in from a csv file\n",
    "bucket = client.get_bucket(project)\n",
    "d_cpt = StringIO(bucket.get_blob('mimic_csv/D_CPT.csv').download_as_string())\n",
    "df = pd.read_csv(d_cpt)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# perform some manipulations and save dataframe back to a file\n",
    "result = df.to_csv()\n",
    "bucket = client.get_bucket(project)\n",
    "# warning, if file already exists, it will be overwritten\n",
    "new_blob = bucket.blob('mimic_csv/new_file.csv')\n",
    "new_blob.upload_from_string(result)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}