pandareen

./spark-shell --master spark://spark-master:7077 --conf "spark.worker.timeout=50000" --conf "spark.shuffle.io.retryWait=6000s" --conf "spark.shuffle.io.maxRetries=1000"

val NUM_SAMPLES=1000000000
val count = sc.parallelize(1 to NUM_SAMPLES).repartition(200).filter { _ =>
  val x = math.random
  val y = math.random
  x*x + y*y < 1
}.count()
println(s"Pi is roughly ${4.0 * count / NUM_SAMPLES}")

pandareen

Prerequisites:

• Operating system:
  • Can be any of Windows/Mac/Linux
• Any one browser of your choice (Chrome/Firefox/Brave)
• Internet connection to download the prerequisites (Will download around 1GB data)
• Free disk space (5GB)
• Docker Desktop
  • Download and install your docker desktop from here https://www.docker.com/
• Start the docker desktop app. Let it start the docker engine. Minimize it.
• Download the Spark + Jupyter combo Docker image jupyter/pyspark-notebook:2023-06-01

pandareen

import paramiko

ssh = paramiko.SSHClient()
ssh.connect('172.17.0.4', username='remoteuser2', password='123456789')
ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command('ls')

pandareen

jenkins_master:
  image: jenkins/jenkins:lts
  container_name: jenkins
  user: root
  environment:
    - TZ=America/Denver
  restart: always
  volumes:
    - /var/run/docker.sock:/var/run/docker.sock
    - /usr/bin/docker:/usr/bin/docker

pandareen

# script to reindex only company
curl --location --request POST 'http://127.0.0.1:9200/_reindex' \
--header 'Content-Type: application/json' \
--data-raw '{
  "source": {
    "index": "seupload",
    "_source": ["job_company_id", "job_company_name", "job_company_website", "job_company_size", "job_company_founded", "job_company_industry", "job_company_linkedin_url", "job_company_linkedin_id", "job_company_facebook_url", "job_company_twitter_url", "job_company_location_name", "job_company_location_locality", "job_company_location_metro", "job_company_location_region", "job_company_location_geo", "job_company_location_street_address", "job_company_location_address_line_2", "job_company_location_postal_code", "job_company_location_country", "job_company_location_continent"]
  },
  "dest": {
    "index": "company"

pandareen

123
das
d
asd
a
sd
asd

pandareen

$instances=(Get-EC2Instance -Filter @{ Name="owner-id";Values=111111111}, @{ Name="tag:environment";Values="xx"}).Instances

$complete_list = @()

forEach ( $ins in $instances ) {
    
    $HostName=($ins.Tags|? {$_.Key -eq "ServerName"});
    $instance_ID=$($ins.instanceId);
    $operations_list += New-Object -TypeName psobject -Property @{HostName=$HostName.Value; InstanceId=$instance_ID}
}

pandareen

$instances=(Get-EC2Instance -Filter @{ Name="owner-id";Values=111111111}, @{ Name="tag:environment";Values="xx"}).Instances

$testing_list = @()
$operations_list = @()

forEach ( $ins in $instances ) {
    
     if ($ins.Tags |? {($_.Key -eq "Component") -and ($_.Value -eq "testing")}) {
        $HostName=($ins.Tags|? {$_.Key -eq "ServerName"});
        $instance_ID=$($ins.instanceId);

pandareen

cd ~/testscripts
docker pull jenkins/jenkins:lts
docker run -p 8080:8080 -p 50000:50000 -v /Users/syapuram/testscripts/:/var/jenkins_home jenkins/jenkins:lts

pandareen

#!/usr/bin/python

import smtplib, ssl

smtp_server = "smtp.ethereal.email"
port = 587  # For starttls
sender_email = "blake.walker81@ethereal.email"
password = "kH4rVhmPWwdkSX4ZvC"

# Create a secure SSL context