berlinbrown

====
When I start a spark-shell from machine B to spark master on B I get this:

scala> sc.master
res3: String = spark://machinB:7077

scala> sc.textFile("/tmp/data/myfile.csv").count()
res4: Long = 976

berlinbrown

>>> y = 0
>>> for x in range(0, 300, 15):
...     print(str(x) + ' season ' + str(y))
...     y = y + 1
...     if (y == 4):
...             y = 0
...

berlinbrown

package pipeline.mockdump;

import com.ibm.mq.*;
import com.ibm.mq.constants.CMQC;

import java.io.IOException;
import java.text.DecimalFormat;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Hashtable;

berlinbrown

package com.primerica.services.clientscaffold.api;
import org.sprig.BootApplication;
@BootApplication
public class ClientScaffoldApplication {
    public void main(final String args) {
        SpringApplication.run(ClientScaffoldApplication.class, args);
    }
}

berlinbrown

# Build stage - 4.9.2020
FROM amazonlinux:2.0.20200304.0 as build-stage
WORKDIR /app
COPY app/ ./
COPY .npmrc ./

# From IBM documentation:
RUN yum install -y add make gcc gcc-c++
RUN yum install -y kernel-devel
RUN yum install -y openssl-devel bzip2-devel

berlinbrown

# Build stage
FROM amazonlinux:latest as build-stage
#FROM node:10 as build-stage
WORKDIR /app
COPY app/ ./
COPY .npmrc ./

# From IBM documentation:
RUN yum install -y add make gcc gcc-c++
RUN yum install -y kernel-devel

berlinbrown

/* Copyright (c) 2013 Berlin Brown (berlin2research.com)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,

berlinbrown

Octane Crawler is a fun/safe/friendly crawler.  I am barelling/requesting at about 10-15 seconds a host.  So, I am gathering about 100 requests a day.

mysql> select count(1) from bot_crawler_links;
+----------+
| count(1) |
+----------+
|     4746 |
+----------+
1 row in set (0.01 sec)

berlinbrown

insert into bot_crawler_seeds(created_at, scheme, host, path, enabled) values('2013-03-14 03:22:36', 'http', 'supremecourt.c-span.org', '/', 'Y');
insert into bot_crawler_seeds(created_at, scheme, host, path, enabled) values('2013-03-14 03:22:36', 'http', 'www.tocqueville.org', '/', 'Y');
insert into bot_crawler_seeds(created_at, scheme, host, path, enabled) values('2013-03-14 03:22:36', 'http', 'whitehouse.c-span.org', '/', 'Y');
insert into bot_crawler_seeds(created_at, scheme, host, path, enabled) values('2013-03-14 03:22:36', 'http', 'mises.givingplan.net', '/', 'Y');
insert into bot_crawler_seeds(created_at, scheme, host, path, enabled) values('2013-03-14 03:22:36', 'http', 'presidentiallibraries.c-span.org', '/', 'Y');
insert into bot_crawler_seeds(created_at, scheme, host, path, enabled) values('2013-03-14 03:22:36', 'http', 'topics.nytimes.com', '/', 'Y');
insert into bot_crawler_seeds(created_at, scheme, host, path, enabled) values('2013-03-14 03:22:36', 'http', 'journalism.nyu.edu', '/', 'Y');
inser

berlinbrown

Here are some popular seeds for basic web crawling.

 www.realclearreligion.org         
 www.michigan.com                  
 www.instapaper.com                
 itunes.apple.com                  
 www.detroitnews.com               
 shop.npr.org                      
 washington.cbslocal.com           
 www.marco.org