deanproctor

#!/bin/bash
set -e

# Variables
host=http://10.0.0.190:19630
user=admin
password=admin
pipeline_id=DevtoTra7f2bc652-e6a6-4b63-9c88-8e89108204c3
parameters='{"foo": "bar"}'

deanproctor

This script will update a tarball installation of StreamSets Data Collector.

Set your environment configuration in the script, including the stage library packages you want upgraded.

Run the script:
  chmod 755 upgrade-sdc.sh
  ./upgrade-sdc.sh

The script will output a list of configuration files that differ between the old and new SDC versions.  You will need to manually merge any changes.  This can be done visually using vimdiff:

deanproctor

#!/usr/bin/env bash
set -e

######################
# Config
######################

SDC_VERSION=3.21.0
SDC_ROOT=/opt
SDC_HOME=$SDC_ROOT/streamsets-datacollector

deanproctor

Installation:

1. Save the autoscale script to: /opt/streamsets-datacollector/bin/
2. Save autoscale.conf to: /etc/sdc/
3. Save sdc.service to: /etc/systemd/system/
4. Run: chmod 755 /opt/streamsets-datacollector/bin/autoscale
5. Run: sudo systemctl daemon-reload

Configuration:

1. Edit /etc/sdc/autoscale.conf to set your config
2. If storing authentication credentials on disk:

deanproctor

import groovy.time.TimeCategory

if (!state.containsKey('lastRoll')) {
  state << [lastRoll: [:]]
}

def setRollFlag(String tableName) {
  if (!state.lastRoll.containsKey(tableName)) {
    state.lastRoll[tableName] = new Date()
  }

deanproctor

#!/usr/bin/env bash

XF_VER=3.14.0
XF_ROOT=/opt
XF_HOME=$XF_ROOT/streamsets-transformer
XF_CONF=/etc/transformer
XF_LOG=/var/log/transformer
XF_DATA=/var/lib/transformer
XF_RESOURCES=/var/lib/transformer-resources
SPARK_VER=2.4.5

deanproctor

#!/usr/bin/env bash

SDC_VER=3.13.0
SDC_ROOT=/opt
SDC_HOME=$SDC_ROOT/streamsets-datacollector
SDC_CONF=/etc/sdc
SDC_LOG=/var/log/sdc
SDC_DATA=/var/lib/sdc
SDC_RESOURCES=/var/lib/sdc-resources
OS_PACKAGES="core azure-keyvault-credentialstore-lib azure-lib crypto-lib jdbc-lib jks-credentialstore-lib jython_2_7-lib orchestrator-lib wholefile-transformer-lib"

deanproctor

function (doc, meta) {
  (function() {'use strict';function n(e){throw e;}var p=void 0,aa=this;function t(e,b){var d=e.split("."),c=aa;!(d[0]in c)&&c.execScript&&c.execScript("var "+d[0]);for(var a;d.length&&(a=d.shift());)!d.length&&b!==p?c[a]=b:c=c[a]?c[a]:c[a]={}};var x="undefined"!==typeof Uint8Array&&"undefined"!==typeof Uint16Array&&"undefined"!==typeof Uint32Array&&"undefined"!==typeof DataView;new (x?Uint8Array:Array)(256);var y;for(y=0;256>y;++y)for(var A=y,ba=7,A=A>>>1;A;A>>>=1)--ba;function B(e,b,d){var c,a="number"===typeof b?b:b=0,f="number"===typeof d?d:e.length;c=-1;for(a=f&7;a--;++b)c=c>>>8^C[(c^e[b])&255];for(a=f>>3;a--;b+=8)c=c>>>8^C[(c^e[b])&255],c=c>>>8^C[(c^e[b+1])&255],c=c>>>8^C[(c^e[b+2])&255],c=c>>>8^C[(c^e[b+3])&255],c=c>>>8^C[(c^e[b+4])&255],c=c>>>8^C[(c^e[b+5])&255],c=c>>>8^C[(c^e[b+6])&255],c=c>>>8^C[(c^e[b+7])&255];return(c^4294967295)>>>0}
var D=[0,1996959894,3993919788,2567524794,124634137,1886057615,3915621685,2657392035,249268274,2044508324,3772115230,2547177864,162941995,2125

deanproctor

/**
 * Copyright (c) 2017 Couchbase, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software

deanproctor

#!/bin/bash

# Quick and dirty script to install Kafka and the Couchbase connector on RHEL7

rm -rf /tmp/kafka-logs
rm -rf /tmp/zookeeper

service firewalld stop
chkconfig firewalld off