andrewstevenson

import json
from optparse import OptionParser
import ConfigParse
from cm_api.api_client import ApiResource
config = ConfigParser.ConfigParser()
 
def load_host_config_files(host_conf_dir):
    with open("%s/host_configuration.json" % host_conf_dir) as cluster_config_file:
        cluster_configs = json.load(cluster_config_file)
    return cluster_configs

andrewstevenson

override def connect(url: String, info: Properties): Connection = {
 
   val connectionProps = Krb5SqlServer.connectionProperties(url)
   val keytabFile = connectionProps(Krb5SqlServer.keytabFile)
   val principal = connectionProps(Krb5SqlServer.principalKey)
 
   val config = new Configuration()
   config.addResource("/etc/hadoop/conf/hdfs-site.xml")
   config.addResource("/etc/hadoop/conf/core-site.xml")
   config.addResource("/etc/hadoop/conf/mapred-site.xml")

andrewstevenson

{
“kb_realm”: “MY_REALM”,
“kdc_master”: “kdc”,
“kdc_admin_user” : “admin”,
“kdc_pass” : “passwod”,
“dr_kdc_master” : “my_backup_kdc”
}

andrewstevenson

#make confluent home folder
mkdir confluent
#download confluent
wget http://packages.confluent.io/archive/3.2/confluent-3.2.0-2.11.tar.gz
#extract archive to confluent folder
tar -xvf confluent-3.2.0-2.11.tar.gz -C confluent
#setup variables
export CONFLUENT_HOME=~/confluent/confluent-3.2.0
cd $CONFLUENT_HOME

andrewstevenson

wget https://github.com/datamountaineer/stream-reactor/releases/download/v0.2.5/stream-reactor-0.2.5-3.2.0.tar.gz
mkdir stream-reactor
tar xvf stream-reactor-0.2.5-3.2.0.tar.gz -C stream-reactor
cd stream-reactor/stream-reactor-0.2.5-3.2.0
bin/start-connect.sh

andrewstevenson

.
|-- acceptance
|   |-- resource-quota.yaml
|   |-- service-cassandra.yaml
|   |-- service-cloudera.yaml
|   |-- service-confluent.yaml
|   `-- service-other.yaml
|-- landscape
|   |-- twitter-sink-azuredocdb.yaml
|   |-- twitter-sink-hdfs.yaml

andrewstevenson

{% set connector_props = env_to_props('CONNECTOR_', '') -%}
{% for name, value in connector_props.iteritems() -%}
{{name}}={{value}}
{% endfor -%}

andrewstevenson

{% set connect_props = env_to_props('CONNECT_', '') -%}
{% for name, value in connect_props.iteritems() -%}
{{name}}={{value}}
{% endfor -%}

andrewstevenson

name: raw-sink-cassandra
release:
  chart: datamountaineer/connector-cassandra:0.1
  version: 0.1.0
configuration:
  clusterName: "raw_sink_cassandra"
  topics: "raw_topic"
  connectCassandraSinkKcql: INSERT INTO data SELECT * FROM raw_topic
  connectCassandraContactPoints: cassandra 
  connectCassandraPort: 9042

andrewstevenson

apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: {{ template "fullname" . }}
  labels:
    app: {{ template "fullname" . }}
    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
    release: "{{ .Release.Name }}"
    heritage: "{{ .Release.Service }}"
spec: