>>> rdd = sc.parallelize([1,2,3])
>>> rdd.map(lambda x: [x,x*10])
RDD: [1,2,3] -> [[1,10], [2,20], [3,30]]
>>> rdd.flatMap(lambda x: [x,x*10])
RDD: [1,2,3] -> [[1,10,2,20,3,30]]
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
find . -type f -exec awk 'END { if (NR > 1) print FILENAME }' {} \; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
function parallel_function { | |
# something | |
} | |
max_parallel_jobs=4 | |
while true | |
do | |
parallel_function & | |
((start_date=start_date+1)) | |
while [ $(jobs | wc -l) -ge $max_parallel_jobs ] ; do sleep 1 ; done |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# List common lines in $1 and $2 | |
function common { | |
comm --check-order -12 $1 $2 | |
} | |
# List lines only present in $1 | |
function onlyina { | |
comm --check-order -3 $1 $2 | sed '/\t/d' | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
input=$1 | |
result="${input/find/replacement}" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
class Singleton(type): | |
_instances = {} | |
def __call__(cls, *args, **kwargs): | |
if cls not in cls._instances: | |
cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs) | |
return cls._instances[cls] | |
#Python2 | |
class MyClass(BaseClass): | |
__metaclass__ = Singleton |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#Git recursive | |
# Apply a command to all sub git directories | |
gr() { | |
rootpath=`pwd`/ | |
cmd=$* | |
if [ $# -eq 0 ]; then | |
echo "Multiline (Add empty line to end the input)" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
awk '{s+=$1} END {printf "%.0f", s}' mydatafile |
Package-1/namespace/__init__.py
Package-1/namespace/module1/__init__.py
Package-2/namespace/__init__.py
Package-2/namespace/module2/__init__.py
with
- Package-1
- Package-2
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
du -hsx * | sort -rh | head -10 |