environment.yml

name: gdaltest
channels:
  - conda-forge
  - defaults
dependencies:
  - _libgcc_mutex=0.1=conda_forge
  - _openmp_mutex=4.5=0_gnu
  - backcall=0.1.0=py_0
  - boost-cpp=1.70.0=h8e57a91_2
  - boto3=1.11.7=py_0
  - botocore=1.14.7=py_0
  - bzip2=1.0.8=h516909a_2
  - ca-certificates=2019.11.28=hecc5488_0
  - cairo=1.16.0=hfb77d84_1002
  - certifi=2019.11.28=py36_0
  - cffi=1.13.2=py36h8022711_0
  - cfitsio=3.470=hb60a0a2_2
  - cryptography=2.8=py36h72c5cf5_1
  - curl=7.65.3=hf8cf82a_0
  - cycler=0.10.0=py_2
  - dbus=1.13.6=he372182_0
  - decorator=4.4.1=py_0
  - docutils=0.15.2=py36_0
  - expat=2.2.5=he1b5a44_1004
  - fontconfig=2.13.1=h86ecdb6_1001
  - freetype=2.10.0=he983fc9_1
  - freexl=1.0.5=h14c3975_1002
  - gdal=2.4.4=py36h5f563d9_0
  - geos=3.8.0=he1b5a44_0
  - geotiff=1.5.1=h38872f0_8
  - gettext=0.19.8.1=hc5be6a0_1002
  - giflib=5.1.7=h516909a_1
  - glib=2.58.3=py36h6f030ca_1002
  - gst-plugins-base=1.14.5=h0935bb2_0
  - gstreamer=1.14.5=h36ae1b5_0
  - hdf4=4.2.13=hf30be14_1003
  - hdf5=1.10.5=nompi_h3c11f04_1104
  - icu=64.2=he1b5a44_1
  - idna=2.8=py36_1000
  - ipython=7.11.1=py36h5ca1d4c_0
  - ipython_genutils=0.2.0=py_1
  - jedi=0.15.2=py36_0
  - jmespath=0.9.4=py_0
  - jpeg=9c=h14c3975_1001
  - json-c=0.13.1=h14c3975_1001
  - kealib=1.4.10=h58c409b_1005
  - kiwisolver=1.1.0=py36hc9558a2_0
  - krb5=1.16.4=h2fd8d38_0
  - libblas=3.8.0=14_openblas
  - libcblas=3.8.0=14_openblas
  - libclang=9.0.1=default_hde54327_0
  - libcurl=7.65.3=hda55be3_0
  - libdap4=3.20.4=hd3bb157_0
  - libedit=3.1.20170329=hf8c457e_1001
  - libffi=3.2.1=he1b5a44_1006
  - libgcc-ng=9.2.0=h24d8f2e_2
  - libgdal=2.4.4=h2b6fda6_0
  - libgfortran-ng=7.3.0=hdf63c60_4
  - libgomp=9.2.0=h24d8f2e_2
  - libiconv=1.15=h516909a_1005
  - libkml=1.3.0=h4fcabce_1010
  - liblapack=3.8.0=14_openblas
  - libllvm9=9.0.1=hc9558a2_0
  - libnetcdf=4.7.3=nompi_h94020b1_100
  - libopenblas=0.3.7=h5ec1e0e_6
  - libpng=1.6.37=hed695b0_0
  - libpq=12.1=hd9ab2ff_0
  - libspatialite=4.3.0a=ha48a99a_1034
  - libssh2=1.8.2=h22169c7_2
  - libstdcxx-ng=9.2.0=hdf63c60_2
  - libtiff=4.1.0=hc3755c2_3
  - libuuid=2.32.1=h14c3975_1000
  - libxcb=1.13=h14c3975_1002
  - libxkbcommon=0.9.1=hebb1f50_0
  - libxml2=2.9.10=hee79883_0
  - lz4-c=1.8.3=he1b5a44_1001
  - matplotlib=3.1.2=py36_1
  - matplotlib-base=3.1.2=py36h250f245_1
  - ncurses=6.1=hf484d3e_1002
  - nspr=4.24=he1b5a44_0
  - nss=3.47=he751ad9_0
  - numpy=1.17.5=py36h95a1406_0
  - olefile=0.46=py_0
  - openjpeg=2.3.1=h981e76c_3
  - openssl=1.1.1d=h516909a_0
  - parso=0.5.2=py_0
  - pcre=8.43=he1b5a44_0
  - pexpect=4.7.0=py36_0
  - pickleshare=0.7.5=py36_1000
  - pillow=7.0.0=py36hefe7db6_0
  - pip=19.3.1=py36_0
  - pixman=0.38.0=h516909a_1003
  - poppler=0.67.0=h14e79db_8
  - poppler-data=0.4.9=1
  - postgresql=12.1=hc63931a_0
  - proj=6.3.0=hc80f0dc_0
  - prompt_toolkit=3.0.2=py_0
  - pthread-stubs=0.4=h14c3975_1001
  - ptyprocess=0.6.0=py_1001
  - pycparser=2.19=py36_1
  - pygments=2.5.2=py_0
  - pyopenssl=19.1.0=py36_0
  - pyparsing=2.4.6=py_0
  - pyqt=5.12.3=py36hcca6a23_1
  - pysocks=1.7.1=py36_0
  - python=3.6.7=h357f687_1006
  - python-dateutil=2.8.1=py_0
  - qt=5.12.5=hd8c4c69_1
  - readline=8.0=hf8c457e_0
  - s3transfer=0.3.1=py36_0
  - setuptools=45.1.0=py36_0
  - six=1.14.0=py36_0
  - sqlite=3.30.1=hcee41ef_0
  - tk=8.6.10=hed695b0_0
  - tornado=6.0.3=py36h516909a_0
  - traitlets=4.3.3=py36_0
  - tzcode=2019a=h516909a_1002
  - urllib3=1.25.7=py36_0
  - wcwidth=0.1.8=py_0
  - wheel=0.33.6=py36_0
  - xerces-c=3.2.2=h8412b87_1004
  - xorg-kbproto=1.0.7=h14c3975_1002
  - xorg-libice=1.0.10=h516909a_0
  - xorg-libsm=1.2.3=h84519dc_1000
  - xorg-libx11=1.6.9=h516909a_0
  - xorg-libxau=1.0.9=h14c3975_0
  - xorg-libxdmcp=1.1.3=h516909a_0
  - xorg-libxext=1.3.4=h516909a_0
  - xorg-libxrender=0.9.10=h516909a_1002
  - xorg-renderproto=0.11.1=h14c3975_1002
  - xorg-xextproto=7.3.0=h14c3975_1002
  - xorg-xproto=7.0.31=h14c3975_1007
  - xz=5.2.4=h14c3975_1001
  - zlib=1.2.11=h516909a_1006
  - zstd=1.4.4=h3b9ef0a_1
  - pip:
    - pyqt5-sip==4.19.18
    - pyqtwebengine==5.12.1
prefix: /home/thomasg/miniconda3/envs/gdaltest

read-s3-geospatial-file-rasterio-gdal-or-boto.py

# Open S3 with Rasterio
import rasterio

s3uri = 's3://mapbox/rasterio/shade.tif'
with rasterio.open(s3uri) as dataset:
    print(dataset.bounds, dataset.count, dataset.indexes) 
    band1 = dataset.read(1) 
    print(band1)


# Open S3 with GDAL only
# Need GDAL 2.3+
# export AWS_SECRET_ACCESS_KEY="TRUE"
from osgeo import gdal

# For s3 streaming, use '/vsis3_streaming/' instead of '/vsis3/'
ds = gdal.Open(s3uri.replace('s3://', '/vsis3/'))

print(ds.RasterCount)
srcband = ds.GetRasterBand(1)
print(srcband)


# Pure python (no geospatial library)
# There more alternative depending wether you want to manipulate image in 
# memory or write it
import boto3
from io import BytesIO
import matplotlib.image as mpimg
import matplotlib.pyplot as plt

resource = boto3.resource('s3')
bucket = resource.Bucket('mapbox')

image_object = bucket.Object('rasterio/shade.tif')
image = mpimg.imread(BytesIO(image_object.get()['Body'].read()), 'tif')

plt.figure(0)
plt.imshow(image)
plt.savefig('shade.png')

requirements.txt

# This file may be used to create an environment using:
# $ conda create --name <env> --file <this file>
# platform: linux-64
_libgcc_mutex=0.1=conda_forge
_openmp_mutex=4.5=0_gnu
backcall=0.1.0=py_0
boost-cpp=1.70.0=h8e57a91_2
boto3=1.11.7=py_0
botocore=1.14.7=py_0
bzip2=1.0.8=h516909a_2
ca-certificates=2019.11.28=hecc5488_0
cairo=1.16.0=hfb77d84_1002
certifi=2019.11.28=py36_0
cffi=1.13.2=py36h8022711_0
cfitsio=3.470=hb60a0a2_2
cryptography=2.8=py36h72c5cf5_1
curl=7.65.3=hf8cf82a_0
cycler=0.10.0=py_2
dbus=1.13.6=he372182_0
decorator=4.4.1=py_0
docutils=0.15.2=py36_0
expat=2.2.5=he1b5a44_1004
fontconfig=2.13.1=h86ecdb6_1001
freetype=2.10.0=he983fc9_1
freexl=1.0.5=h14c3975_1002
gdal=2.4.4=py36h5f563d9_0
geos=3.8.0=he1b5a44_0
geotiff=1.5.1=h38872f0_8
gettext=0.19.8.1=hc5be6a0_1002
giflib=5.1.7=h516909a_1
glib=2.58.3=py36h6f030ca_1002
gst-plugins-base=1.14.5=h0935bb2_0
gstreamer=1.14.5=h36ae1b5_0
hdf4=4.2.13=hf30be14_1003
hdf5=1.10.5=nompi_h3c11f04_1104
icu=64.2=he1b5a44_1
idna=2.8=py36_1000
ipython=7.11.1=py36h5ca1d4c_0
ipython_genutils=0.2.0=py_1
jedi=0.15.2=py36_0
jmespath=0.9.4=py_0
jpeg=9c=h14c3975_1001
json-c=0.13.1=h14c3975_1001
kealib=1.4.10=h58c409b_1005
kiwisolver=1.1.0=py36hc9558a2_0
krb5=1.16.4=h2fd8d38_0
libblas=3.8.0=14_openblas
libcblas=3.8.0=14_openblas
libclang=9.0.1=default_hde54327_0
libcurl=7.65.3=hda55be3_0
libdap4=3.20.4=hd3bb157_0
libedit=3.1.20170329=hf8c457e_1001
libffi=3.2.1=he1b5a44_1006
libgcc-ng=9.2.0=h24d8f2e_2
libgdal=2.4.4=h2b6fda6_0
libgfortran-ng=7.3.0=hdf63c60_4
libgomp=9.2.0=h24d8f2e_2
libiconv=1.15=h516909a_1005
libkml=1.3.0=h4fcabce_1010
liblapack=3.8.0=14_openblas
libllvm9=9.0.1=hc9558a2_0
libnetcdf=4.7.3=nompi_h94020b1_100
libopenblas=0.3.7=h5ec1e0e_6
libpng=1.6.37=hed695b0_0
libpq=12.1=hd9ab2ff_0
libspatialite=4.3.0a=ha48a99a_1034
libssh2=1.8.2=h22169c7_2
libstdcxx-ng=9.2.0=hdf63c60_2
libtiff=4.1.0=hc3755c2_3
libuuid=2.32.1=h14c3975_1000
libxcb=1.13=h14c3975_1002
libxkbcommon=0.9.1=hebb1f50_0
libxml2=2.9.10=hee79883_0
lz4-c=1.8.3=he1b5a44_1001
matplotlib=3.1.2=py36_1
matplotlib-base=3.1.2=py36h250f245_1
ncurses=6.1=hf484d3e_1002
nspr=4.24=he1b5a44_0
nss=3.47=he751ad9_0
numpy=1.17.5=py36h95a1406_0
olefile=0.46=py_0
openjpeg=2.3.1=h981e76c_3
openssl=1.1.1d=h516909a_0
parso=0.5.2=py_0
pcre=8.43=he1b5a44_0
pexpect=4.7.0=py36_0
pickleshare=0.7.5=py36_1000
pillow=7.0.0=py36hefe7db6_0
pip=19.3.1=py36_0
pixman=0.38.0=h516909a_1003
poppler=0.67.0=h14e79db_8
poppler-data=0.4.9=1
postgresql=12.1=hc63931a_0
proj=6.3.0=hc80f0dc_0
prompt_toolkit=3.0.2=py_0
pthread-stubs=0.4=h14c3975_1001
ptyprocess=0.6.0=py_1001
pycparser=2.19=py36_1
pygments=2.5.2=py_0
pyopenssl=19.1.0=py36_0
pyparsing=2.4.6=py_0
pyqt=5.12.3=py36hcca6a23_1
pyqt5-sip=4.19.18=pypi_0
pyqtwebengine=5.12.1=pypi_0
pysocks=1.7.1=py36_0
python=3.6.7=h357f687_1006
python-dateutil=2.8.1=py_0
qt=5.12.5=hd8c4c69_1
readline=8.0=hf8c457e_0
s3transfer=0.3.1=py36_0
setuptools=45.1.0=py36_0
six=1.14.0=py36_0
sqlite=3.30.1=hcee41ef_0
tk=8.6.10=hed695b0_0
tornado=6.0.3=py36h516909a_0
traitlets=4.3.3=py36_0
tzcode=2019a=h516909a_1002
urllib3=1.25.7=py36_0
wcwidth=0.1.8=py_0
wheel=0.33.6=py36_0
xerces-c=3.2.2=h8412b87_1004
xorg-kbproto=1.0.7=h14c3975_1002
xorg-libice=1.0.10=h516909a_0
xorg-libsm=1.2.3=h84519dc_1000
xorg-libx11=1.6.9=h516909a_0
xorg-libxau=1.0.9=h14c3975_0
xorg-libxdmcp=1.1.3=h516909a_0
xorg-libxext=1.3.4=h516909a_0
xorg-libxrender=0.9.10=h516909a_1002
xorg-renderproto=0.11.1=h14c3975_1002
xorg-xextproto=7.3.0=h14c3975_1002
xorg-xproto=7.0.31=h14c3975_1007
xz=5.2.4=h14c3975_1001
zlib=1.2.11=h516909a_1006
zstd=1.4.4=h3b9ef0a_1