python_utilities

hillslope_delineation.py

from seims.preprocess.sd_hillslope import DelineateHillslope
streamf = r'stream_link.tif'
flowdirf = r'flow_dir.tif'
hillslpf = r'hillslope.tif'
import time
s = time.time()
DelineateHillslope.downstream_method_whitebox(streamf2, flowdirf, hillslpf, "taudem", 0)
e = time.time()
print ("time consuming: %.2fs" % (e - s))

numpy_get_unique_values_and_counts

# https://stackoverflow.com/questions/10741346/numpy-most-efficient-frequency-counts-for-unique-values-in-an-array
import numpy as np

x = np.array([1,1,1,2,2,2,5,25,1,1])
unique, counts = np.unique(x, return_counts=True)

print np.asarray((unique, counts)).T

# which gives:
# [[ 1  5]
#  [ 2  3]
#  [ 5  1]
#  [25  1]]

RelativeImportModules.py

from pathlib2 import Path


# Supplement for http://stackoverflow.com/a/28154841/2301450
# Generic boilerplate code for setting __package__ attribute for relative imports
# Code from https://gist.github.com/vaultah/d63cb4c86be2774377aa674b009f759a
# Usage:
#     1_level_example
#
#         if __name__ == '__main__' and __package__ is None:
#             __package__ import_parents()
#
#         from . import module
#         from .module.submodule import thing
#     3_level_example
#
#         if __name__ == '__main__' and __package__ is None:
#             __package__ = import_parents(level=3)
#
#         from ... import module
#         from ...module.submodule import thing

def import_parents(level=1):
    global __package__
    file = Path(__file__).resolve()
    parent, top = file.parent, file.parents[level]
    __package__ = '.'.join(parent.parts[len(top.parts):])
    sys.path.append(str(top))
    importlib.import_module(__package__)  # won't be needed after that
    return __package__


if __name__ == '__main__' and __package__ is None:
    import_parents(level=1)

scoop_submit_qsub.sh

#!/bin/sh -f
#PBS -l nodes=4:ppn=4
#PBS -V
#PBS -N SCOOPJob

# Path to your executable. For example, if you extracted SCOOP to $HOME/downloads/scoop
cd $HOME/test

# Add any addition to your environment variables like PATH. For example, if your local python installation is in $HOME/python
#export PATH=$HOME/python/bin:$PATH

# If, instead, you are using the python offered by the system, you can stipulate it's library path via PYTHONPATH
#export PYTHONPATH=$HOME/wanted/path/lib/python+version/site-packages/:$PYTHONPATH
# Or use VirtualEnv via virtualenvwrapper here:
#workon yourenvironment

# workers is default to nodes * ppn, e.g., #PBS -l nodes=4:ppn=4 is the same to '-n 16'
python -m scoop -vv map_reduce.py > SCOOPJob.stdout.log