One Paragraph of project description goes here
These instructions will get you a copy of the project up and running on your local machine for development and testing purposes. See deployment for notes on how to deploy the project on a live system.
import org.apache.spark._ | |
import org.apache.spark.graphx._ | |
import java.io.File | |
import java.time._ | |
import java.util.Calendar | |
import scala.util.MurmurHash | |
import java.io.PrintWriter | |
import org.apache.spark.rdd.RDD |
import org.elasticsearch.hadoop._ | |
import org.elasticsearch.spark._ | |
import java.time._ | |
import java.time.format._ | |
// Get parameters | |
// args(0) = Namenode IP Address ( Resource Manager on MapR distribution) | |
// args(1) = nutch segment name | |
// args(2) = Elasticsearch index name | |
val args = sc.getConf.get("spark.driver.args").split("\\s+") |
#!/usr/bin/env bash | |
## default variables values ## | |
## TODO: ansible_mirror #### | |
## TODO: des_copy == src_install ## | |
## refers parameter expansion : https://opensource.com/article/17/6/bash-parameter-expansion?sc_cid=70160000001273HAAQ ### | |
mirror=ftp://195.220.108.108/linux | |
docker_mirror=https://yum.dockerproject.org/repo/main/centos/7/Packages | |
dest_copy=${HOME}/ansible/Packages |
## Reference: https://stackoverflow.com/questions/1967370/git-replacing-lf-with-crlf | |
git config --local core.autocrlf input |
FROM centos:latest | |
LABEL maintainer Bill METANGMO @billmetangmo github \ | |
description="Optimized for HPC machine learning & Graph processing apps" | |
#dependencies="bash,procps,openjdk8-jre-base,openssh,ca-certificates"\ | |
#external="scala" | |
# proxy are not defined as env variable as thi image would be used in client env where our proxy doesn't make sense | |
ARG http_proxy= | |
ARG https_proxy= | |
ARG no_proxy=localhost,127.0.0.1 |
--- | |
- hosts: registry | |
become: true | |
vars: | |
public_ip: 10.197.138.130 | |
tasks: | |
- name: set selinux to permissive | |
selinux: | |
policy: targeted |
// Iterate through all the zip files in multi-part form-data (memory or disk) | |
// For each file, extract it to destDir and change permissions of the file to the user one | |
// If the zip file is a directory , empty subdirectories are ignored | |
func extractFilesWithUserRights(files []*zip.File, filePath string, username string) error { | |
uid, gid, err := getIdentifier(username) | |
if err != nil { | |
return err | |
} |
def binary_search(alist, item): | |
first = 0 | |
last = len(alist) - 1 | |
found = False | |
while first <= last and not found: | |
midpoint = (first + last) / 2 | |
if alist[midpoint] >= item: | |
found = True | |
else: |
return value.encode("utf-8").decode('raw-unicode-escape') |