EMR has components running on internal ip's such as ip-172-23-53-101.ec2.internal. To resolve to the implied public/vpn IP's, setup dnsmasq
$ brew install dnsmasq
version: '3' | |
services: | |
zookeeper: | |
image: zookeeper | |
kafka: | |
image: wurstmeister/kafka | |
environment: | |
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 | |
KAFKA_ADVERTISED_HOST_NAME: kafka | |
bash: |
# Add the following lines to /usr/local/etc/dnsmasq.conf: | |
address=/ec2.internal/127.0.0.1 | |
synth-domain=ec2.internal,0.0.0.0,255.255.255.255,ip- | |
address=/us-west-2.compute.internal/127.0.0.1 | |
synth-domain=us-west-2.compute.internal,0.0.0.0,255.255.255.255,ip- | |
address=/eu-west-1.compute.internal/127.0.0.1 | |
synth-domain=eu-west-1.compute.internal,0.0.0.0,255.255.255.255,ip- |
import org.apache.spark.sql.DataFrame | |
def dataFrameToDDL(dataFrame: DataFrame, tableName: String): String = { | |
val columns = dataFrame.schema.map { field => | |
" " + field.name + " " + field.dataType.simpleString.toUpperCase | |
} | |
s"CREATE TABLE $tableName (\n${columns.mkString(",\n")}\n)" | |
} | |
import spark.sqlContext.implicits._ |
CREATE TABLE bookdata ( | |
id serial NOT NULL PRIMARY KEY, | |
info json NOT NULL | |
) | |
CREATE INDEX bookdata_fts ON bookdata USING gin((to_tsvector('english',info->'title'))); | |
INSERT INTO bookdata (info) | |
VALUES | |
( '{ "title": "The Tattooed Duke", "items": {"product": "Diaper","qty": 24}}'), |
export COROOT=$GOPATH/src/github.com/crunchydata/postgres-operator | |
export PATH=$PATH:$GOPATH/bin | |
export PGO_CA_CERT=$COROOT/conf/apiserver/server.crt | |
export PGO_CLIENT_CERT=$COROOT/conf/apiserver/server.crt | |
export PGO_CLIENT_KEY=$COROOT/conf/apiserver/server.key | |
export CO_NAMESPACE=demo | |
export CO_CMD=kubectl | |
export CO_APISERVER_URL=https://127.0.0.1:8443 |
#!/bin/bash -eu | |
export GOPATH=$HOME/pgcluster-test | |
export PATH=$PATH:$GOPATH/bin | |
mkdir -p $GOPATH/src $GOPATH/bin $GOPATH/pkg | |
mkdir -p $GOPATH/src/github.com/crunchydata/ | |
# Download PostgreSQL Operator source: | |
cd $GOPATH/src/github.com/crunchydata | |
git clone https://github.com/CrunchyData/postgres-operator.git |
FROM rabbitmq:management | |
RUN rabbitmq-plugins enable --offline rabbitmq_shovel && rabbitmq-plugins enable --offline rabbitmq_shovel_management |
{"log":"time=\"2017-09-22T17:50:58Z\" level=info msg=\"Establishing connection with Rethinkdb\" \n","stream":"stderr","time":"2017-09-22T17:50:58.428872932Z"} | |
{"log":"time=\"2017-09-22T17:50:58Z\" level=info msg=\"writing out configs and watching for changes\" \n","stream":"stderr","time":"2017-09-22T17:50:58.48144991Z"} | |
{"log":"time=\"2017-09-22T17:50:58Z\" level=info msg=\"watching configs\" \n","stream":"stderr","time":"2017-09-22T17:50:58.481480667Z"} | |
{"log":"time=\"2017-09-22T17:50:58Z\" level=info msg=\"watching for changes to configtracker.configSpec{src:\\\"certs/postgres_ca/cert.pem\\\", writer:(configtracker.WriterFunc)(0x4863a0), templateFunc:(configtracker.TemplateFunc)(0x484cb0), cacheKey:\\\"21f500d9-8562-48d6-98db-65b3792665eb\\\"}\" \n","stream":"stderr","time":"2017-09-22T17:50:58.481520965Z"} | |
{"log":"time=\"2017-09-22T17:50:58Z\" level=info msg=\"watching for changes to configtracker.configSpec{src:\\\"certs/postgres_server/cert.pem/a2382528a00b\\\", writer:(configtracker.WriterFunc)(0x4863a |