Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
{
"pipelineConfig" : {
"schemaVersion" : 6,
"version" : 12,
"pipelineId" : "Metrics635c20ed-872c-48d7-87fb-7043ab937ae6",
"title" : "Metrics",
"description" : "",
"uuid" : "08bce59b-151b-47d4-97dc-5d4f58da9672",
"configuration" : [ {
"name" : "executionMode",
"value" : "STANDALONE"
}, {
"name" : "deliveryGuarantee",
"value" : "AT_LEAST_ONCE"
}, {
"name" : "shouldRetry",
"value" : true
}, {
"name" : "retryAttempts",
"value" : -1
}, {
"name" : "notifyOnStates",
"value" : [ "RUN_ERROR", "STOPPED", "FINISHED" ]
}, {
"name" : "emailIDs",
"value" : [ ]
}, {
"name" : "constants",
"value" : [ {
"key" : "JDBC_URI",
"value" : "jdbc:mysql://mysql.example.com/dbname"
}, {
"key" : "JDBC_USER",
"value" : ""
}, {
"key" : "JDBC_PASSWORD",
"value" : ""
}, {
"key" : "AWS_KEY",
"value" : ""
}, {
"key" : "AWS_SECRET",
"value" : ""
}, {
"key" : "KICKFIRE_KEY",
"value" : ""
} ]
}, {
"name" : "badRecordsHandling",
"value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_recordstolocalfilesystem_ToErrorLocalFSDTarget::1"
}, {
"name" : "clusterSlaveMemory",
"value" : 1024
}, {
"name" : "clusterSlaveJavaOpts",
"value" : "-XX:+UseConcMarkSweepGC -XX:+UseParNewGC -Dlog4j.debug"
}, {
"name" : "clusterLauncherEnv",
"value" : [ ]
}, {
"name" : "mesosDispatcherURL",
"value" : null
}, {
"name" : "hdfsS3ConfDir",
"value" : null
}, {
"name" : "rateLimit",
"value" : 0
}, {
"name" : "maxRunners",
"value" : 0
}, {
"name" : "webhookConfigs",
"value" : [ ]
}, {
"name" : "statsAggregatorStage",
"value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_StatsDpmDirectlyDTarget::1"
}, {
"name" : "workerCount",
"value" : 0
}, {
"name" : "startEventStage",
"value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1"
}, {
"name" : "stopEventStage",
"value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1"
}, {
"name" : "errorRecordPolicy",
"value" : "ORIGINAL_RECORD"
}, {
"name" : "shouldCreateFailureSnapshot",
"value" : true
}, {
"name" : "sparkConfigs",
"value" : [ ]
}, {
"name" : "edgeHttpUrl",
"value" : "http://localhost:18633"
}, {
"name" : "runnerIdleTIme",
"value" : 60
}, {
"name" : "testOriginStage",
"value" : "streamsets-datacollector-dev-lib::com_streamsets_pipeline_stage_devtest_rawdata_RawDataDSource::3"
}, {
"name" : "logLevel",
"value" : "INFO"
}, {
"name" : "amazonEMRConfig.userRegion",
"value" : null
}, {
"name" : "amazonEMRConfig.userRegionCustom",
"value" : null
}, {
"name" : "amazonEMRConfig.accessKey",
"value" : ""
}, {
"name" : "amazonEMRConfig.secretKey",
"value" : ""
}, {
"name" : "amazonEMRConfig.s3StagingUri",
"value" : null
}, {
"name" : "amazonEMRConfig.provisionNewCluster",
"value" : false
}, {
"name" : "amazonEMRConfig.clusterId",
"value" : null
}, {
"name" : "amazonEMRConfig.clusterPrefix",
"value" : null
}, {
"name" : "amazonEMRConfig.terminateCluster",
"value" : false
}, {
"name" : "amazonEMRConfig.loggingEnabled",
"value" : true
}, {
"name" : "amazonEMRConfig.s3LogUri",
"value" : null
}, {
"name" : "amazonEMRConfig.enableEMRDebugging",
"value" : true
}, {
"name" : "amazonEMRConfig.serviceRole",
"value" : "EMR_DefaultRole"
}, {
"name" : "amazonEMRConfig.jobFlowRole",
"value" : "EMR_EC2_DefaultRole"
}, {
"name" : "amazonEMRConfig.visibleToAllUsers",
"value" : true
}, {
"name" : "amazonEMRConfig.ec2SubnetId",
"value" : null
}, {
"name" : "amazonEMRConfig.masterSecurityGroup",
"value" : null
}, {
"name" : "amazonEMRConfig.slaveSecurityGroup",
"value" : null
}, {
"name" : "amazonEMRConfig.instanceCount",
"value" : 2
}, {
"name" : "amazonEMRConfig.masterInstanceType",
"value" : null
}, {
"name" : "amazonEMRConfig.masterInstanceTypeCustom",
"value" : null
}, {
"name" : "amazonEMRConfig.slaveInstanceType",
"value" : null
}, {
"name" : "amazonEMRConfig.slaveInstanceTypeCustom",
"value" : null
} ],
"uiInfo" : {
"previewConfig" : {
"previewSource" : "CONFIGURED_SOURCE",
"batchSize" : 10,
"timeout" : "100000",
"writeToDestinations" : false,
"showHeader" : false,
"showFieldType" : true,
"rememberMe" : false
}
},
"fragments" : [ ],
"stages" : [ {
"instanceName" : "AmazonS3_01",
"library" : "streamsets-datacollector-aws-lib",
"stageName" : "com_streamsets_pipeline_stage_origin_s3_AmazonS3DSource",
"stageVersion" : "11",
"configuration" : [ {
"name" : "s3ConfigBean.basicConfig.maxBatchSize",
"value" : 1000
}, {
"name" : "s3ConfigBean.basicConfig.maxWaitTime",
"value" : 2000
}, {
"name" : "s3ConfigBean.sseConfig.useCustomerSSEKey",
"value" : false
}, {
"name" : "s3ConfigBean.sseConfig.customerKey",
"value" : ""
}, {
"name" : "s3ConfigBean.sseConfig.customerKeyMd5",
"value" : ""
}, {
"name" : "s3ConfigBean.proxyConfig.useProxy",
"value" : false
}, {
"name" : "s3ConfigBean.proxyConfig.proxyHost",
"value" : null
}, {
"name" : "s3ConfigBean.proxyConfig.proxyPort",
"value" : null
}, {
"name" : "s3ConfigBean.proxyConfig.proxyUser",
"value" : ""
}, {
"name" : "s3ConfigBean.proxyConfig.proxyPassword",
"value" : ""
}, {
"name" : "s3ConfigBean.errorConfig.errorHandlingOption",
"value" : "NONE"
}, {
"name" : "s3ConfigBean.errorConfig.archivingOption",
"value" : "MOVE_TO_PREFIX"
}, {
"name" : "s3ConfigBean.errorConfig.errorPrefix",
"value" : null
}, {
"name" : "s3ConfigBean.errorConfig.errorBucket",
"value" : null
}, {
"name" : "s3ConfigBean.postProcessingConfig.postProcessing",
"value" : "NONE"
}, {
"name" : "s3ConfigBean.postProcessingConfig.archivingOption",
"value" : "MOVE_TO_PREFIX"
}, {
"name" : "s3ConfigBean.postProcessingConfig.postProcessPrefix",
"value" : null
}, {
"name" : "s3ConfigBean.postProcessingConfig.postProcessBucket",
"value" : null
}, {
"name" : "s3ConfigBean.s3FileConfig.prefixPattern",
"value" : "*.log.gz"
}, {
"name" : "s3ConfigBean.s3FileConfig.objectOrdering",
"value" : "TIMESTAMP"
}, {
"name" : "s3ConfigBean.s3FileConfig.poolSize",
"value" : 100
}, {
"name" : "s3ConfigBean.s3FileConfig.overrunLimit",
"value" : 128
}, {
"name" : "s3ConfigBean.s3Config.bucket",
"value" : "streamsets-downloadlogs"
}, {
"name" : "s3ConfigBean.s3Config.awsConfig.awsAccessKeyId",
"value" : "${AWS_KEY}"
}, {
"name" : "s3ConfigBean.s3Config.awsConfig.awsSecretAccessKey",
"value" : "${AWS_SECRET}"
}, {
"name" : "s3ConfigBean.s3Config.region",
"value" : "US_EAST_1"
}, {
"name" : "s3ConfigBean.s3Config.endpoint",
"value" : null
}, {
"name" : "s3ConfigBean.s3Config.commonPrefix",
"value" : "fastly-cdn"
}, {
"name" : "s3ConfigBean.s3Config.delimiter",
"value" : "/"
}, {
"name" : "s3ConfigBean.enableMetaData",
"value" : false
}, {
"name" : "stageOnRecordError",
"value" : "STOP_PIPELINE"
}, {
"name" : "s3ConfigBean.proxyConfig.connectionTimeout",
"value" : 10
}, {
"name" : "s3ConfigBean.proxyConfig.socketTimeout",
"value" : 50
}, {
"name" : "s3ConfigBean.proxyConfig.retryCount",
"value" : 3
}, {
"name" : "s3ConfigBean.numberOfThreads",
"value" : 1
} ],
"uiInfo" : {
"description" : "",
"label" : "Fastly CDN logs",
"xPos" : 60,
"yPos" : 48.408103942871094,
"stageType" : "SOURCE"
},
"inputLanes" : [ ],
"outputLanes" : [ "AmazonS3_01OutputLane14974564326410" ],
"eventLanes" : [ ],
"services" : [ {
"service" : "com.streamsets.pipeline.api.service.dataformats.DataFormatParserService",
"serviceVersion" : 1,
"configuration" : [ {
"name" : "dataFormat",
"value" : "LOG"
}, {
"name" : "dataFormatConfig.compression",
"value" : "COMPRESSED_FILE"
}, {
"name" : "dataFormatConfig.filePatternInArchive",
"value" : "*"
}, {
"name" : "dataFormatConfig.charset",
"value" : "UTF-8"
}, {
"name" : "dataFormatConfig.removeCtrlChars",
"value" : false
}, {
"name" : "dataFormatConfig.textMaxLineLen",
"value" : 1024
}, {
"name" : "dataFormatConfig.useCustomDelimiter",
"value" : false
}, {
"name" : "dataFormatConfig.customDelimiter",
"value" : "\\r\\n"
}, {
"name" : "dataFormatConfig.includeCustomDelimiterInTheText",
"value" : false
}, {
"name" : "dataFormatConfig.jsonContent",
"value" : "MULTIPLE_OBJECTS"
}, {
"name" : "dataFormatConfig.jsonMaxObjectLen",
"value" : 4096
}, {
"name" : "dataFormatConfig.csvFileFormat",
"value" : "CSV"
}, {
"name" : "dataFormatConfig.csvHeader",
"value" : "NO_HEADER"
}, {
"name" : "dataFormatConfig.csvMaxObjectLen",
"value" : 1024
}, {
"name" : "dataFormatConfig.csvCustomDelimiter",
"value" : "|"
}, {
"name" : "dataFormatConfig.csvCustomEscape",
"value" : "\\"
}, {
"name" : "dataFormatConfig.csvCustomQuote",
"value" : "\""
}, {
"name" : "dataFormatConfig.csvEnableComments",
"value" : false
}, {
"name" : "dataFormatConfig.csvCommentMarker",
"value" : "#"
}, {
"name" : "dataFormatConfig.csvIgnoreEmptyLines",
"value" : true
}, {
"name" : "dataFormatConfig.csvRecordType",
"value" : "LIST_MAP"
}, {
"name" : "dataFormatConfig.csvSkipStartLines",
"value" : 0
}, {
"name" : "dataFormatConfig.parseNull",
"value" : false
}, {
"name" : "dataFormatConfig.nullConstant",
"value" : "\\\\N"
}, {
"name" : "dataFormatConfig.xmlRecordElement",
"value" : null
}, {
"name" : "dataFormatConfig.includeFieldXpathAttributes",
"value" : false
}, {
"name" : "dataFormatConfig.xPathNamespaceContext",
"value" : [ ]
}, {
"name" : "dataFormatConfig.outputFieldAttributes",
"value" : false
}, {
"name" : "dataFormatConfig.xmlMaxObjectLen",
"value" : 4096
}, {
"name" : "dataFormatConfig.logMode",
"value" : "GROK"
}, {
"name" : "dataFormatConfig.logMaxObjectLen",
"value" : 1024
}, {
"name" : "dataFormatConfig.retainOriginalLine",
"value" : false
}, {
"name" : "dataFormatConfig.customLogFormat",
"value" : "%h %l %u %t \"%r\" %>s %b"
}, {
"name" : "dataFormatConfig.regex",
"value" : "^(\\S+) (\\S+) (\\S+) \\[([\\w:/]+\\s[+\\-]\\d{4})\\] \"(\\S+) (\\S+) (\\S+)\" (\\d{3}) (\\d+)"
}, {
"name" : "dataFormatConfig.fieldPathsToGroupName",
"value" : [ {
"fieldPath" : "/",
"group" : 1
} ]
}, {
"name" : "dataFormatConfig.grokPatternDefinition",
"value" : "DATESTAMP_FASTLY %{DAY}, %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} %{WORD:tz}\nSIZE_OR_NULL (?:%{NUMBER:size}|\\(null\\))"
}, {
"name" : "dataFormatConfig.grokPattern",
"value" : "<%{NUMBER:priority}>%{TIMESTAMP_ISO8601:timestamp} %{HOSTNAME:cachenode} %{WORD:logname}\\[%{NUMBER:pid}\\]: %{IP:ip} \"-\" \"-\" %{DATESTAMP_FASTLY:datestamp} %{WORD:verb} %{NOTSPACE:file} %{NUMBER:code} %{SIZE_OR_NULL}"
}, {
"name" : "dataFormatConfig.onParseError",
"value" : "ERROR"
}, {
"name" : "dataFormatConfig.maxStackTraceLines",
"value" : 50
}, {
"name" : "dataFormatConfig.enableLog4jCustomLogFormat",
"value" : false
}, {
"name" : "dataFormatConfig.log4jCustomLogFormat",
"value" : "%r [%t] %-5p %c %x - %m%n"
}, {
"name" : "dataFormatConfig.avroSchemaSource",
"value" : null
}, {
"name" : "dataFormatConfig.avroSchema",
"value" : null
}, {
"name" : "dataFormatConfig.schemaRegistryUrls",
"value" : [ ]
}, {
"name" : "dataFormatConfig.schemaLookupMode",
"value" : "SUBJECT"
}, {
"name" : "dataFormatConfig.subject",
"value" : null
}, {
"name" : "dataFormatConfig.schemaId",
"value" : null
}, {
"name" : "dataFormatConfig.protoDescriptorFile",
"value" : null
}, {
"name" : "dataFormatConfig.messageType",
"value" : null
}, {
"name" : "dataFormatConfig.isDelimited",
"value" : true
}, {
"name" : "dataFormatConfig.binaryMaxObjectLen",
"value" : 1024
}, {
"name" : "dataFormatConfig.datagramMode",
"value" : "SYSLOG"
}, {
"name" : "dataFormatConfig.typesDbPath",
"value" : null
}, {
"name" : "dataFormatConfig.convertTime",
"value" : false
}, {
"name" : "dataFormatConfig.excludeInterval",
"value" : true
}, {
"name" : "dataFormatConfig.authFilePath",
"value" : null
}, {
"name" : "dataFormatConfig.wholeFileMaxObjectLen",
"value" : 8192
}, {
"name" : "dataFormatConfig.rateLimit",
"value" : "-1"
}, {
"name" : "dataFormatConfig.verifyChecksum",
"value" : false
}, {
"name" : "dataFormatConfig.csvAllowExtraColumns",
"value" : false
}, {
"name" : "dataFormatConfig.csvExtraColumnPrefix",
"value" : "_extra_"
}, {
"name" : "dataFormatConfig.netflowOutputValuesMode",
"value" : "RAW_AND_INTERPRETED"
}, {
"name" : "dataFormatConfig.maxTemplateCacheSize",
"value" : -1
}, {
"name" : "dataFormatConfig.templateCacheTimeoutMs",
"value" : -1
}, {
"name" : "dataFormatConfig.netflowOutputValuesModeDatagram",
"value" : "RAW_AND_INTERPRETED"
}, {
"name" : "dataFormatConfig.maxTemplateCacheSizeDatagram",
"value" : -1
}, {
"name" : "dataFormatConfig.templateCacheTimeoutMsDatagram",
"value" : -1
} ]
} ]
}, {
"instanceName" : "FieldTypeConverter_01",
"library" : "streamsets-datacollector-basic-lib",
"stageName" : "com_streamsets_pipeline_stage_processor_fieldtypeconverter_FieldTypeConverterDProcessor",
"stageVersion" : "2",
"configuration" : [ {
"name" : "convertBy",
"value" : "BY_FIELD"
}, {
"name" : "fieldTypeConverterConfigs",
"value" : [ {
"fields" : [ "/size", "/pid", "/code" ],
"targetType" : "INTEGER",
"treatInputFieldAsDate" : false,
"dataLocale" : "en,US",
"scale" : -1,
"decimalScaleRoundingStrategy" : "ROUND_UNNECESSARY",
"dateFormat" : "YYYY_MM_DD",
"encoding" : "UTF-8",
"zonedDateTimeFormat" : "ISO_ZONED_DATE_TIME",
"otherZonedDateTimeFormat" : ""
}, {
"fields" : [ "/timestamp" ],
"targetType" : "DATETIME",
"treatInputFieldAsDate" : false,
"dataLocale" : "en,US",
"scale" : -1,
"decimalScaleRoundingStrategy" : "ROUND_UNNECESSARY",
"dateFormat" : "OTHER",
"encoding" : "UTF-8",
"otherDateFormat" : "yyyy-MM-dd'T'HH:mm:ss'Z'",
"zonedDateTimeFormat" : "ISO_ZONED_DATE_TIME",
"otherZonedDateTimeFormat" : ""
} ]
}, {
"name" : "wholeTypeConverterConfigs",
"value" : [ {
"sourceType" : "INTEGER",
"targetType" : "INTEGER",
"treatInputFieldAsDate" : false,
"dataLocale" : "en,US",
"scale" : -1,
"decimalScaleRoundingStrategy" : "ROUND_UNNECESSARY",
"dateFormat" : "YYYY_MM_DD",
"encoding" : "UTF-8",
"zonedDateTimeFormat" : "ISO_ZONED_DATE_TIME",
"otherZonedDateTimeFormat" : ""
} ]
}, {
"name" : "stageOnRecordError",
"value" : "TO_ERROR"
}, {
"name" : "stageRequiredFields",
"value" : [ ]
}, {
"name" : "stageRecordPreconditions",
"value" : [ ]
} ],
"uiInfo" : {
"description" : "",
"label" : "Convert date, integers",
"xPos" : 280,
"yPos" : 50,
"stageType" : "PROCESSOR"
},
"inputLanes" : [ "AmazonS3_01OutputLane14974564326410" ],
"outputLanes" : [ "FieldTypeConverter_01OutputLane14974687038250" ],
"eventLanes" : [ ],
"services" : [ ]
}, {
"instanceName" : "StreamSelector_01",
"library" : "streamsets-datacollector-basic-lib",
"stageName" : "com_streamsets_pipeline_stage_processor_selector_SelectorDProcessor",
"stageVersion" : "1",
"configuration" : [ {
"name" : "lanePredicates",
"value" : [ {
"outputLane" : "StreamSelector_01OutputLane1499904663845",
"predicate" : "${str:endsWith(record:value('/file'), '.parcel') || str:endsWith(record:value('/file'), '.zip') || str:endsWith(record:value('/file'), '.rpm') || str:endsWith(record:value('/file'), '.tgz')}"
}, {
"outputLane" : "StreamSelector_01OutputLane1499903680944",
"predicate" : "${true}"
}, {
"outputLane" : "StreamSelector_01OutputLane1499903546028",
"predicate" : "default"
} ]
}, {
"name" : "stageOnRecordError",
"value" : "TO_ERROR"
}, {
"name" : "stageRequiredFields",
"value" : [ ]
}, {
"name" : "stageRecordPreconditions",
"value" : [ ]
} ],
"uiInfo" : {
"description" : "",
"label" : "File download?",
"xPos" : 500,
"yPos" : 140,
"stageType" : "PROCESSOR"
},
"inputLanes" : [ "FieldTypeConverter_01OutputLane14974687038250" ],
"outputLanes" : [ "StreamSelector_01OutputLane1499904663845", "StreamSelector_01OutputLane1499903680944", "StreamSelector_01OutputLane1499903546028" ],
"eventLanes" : [ ],
"services" : [ ]
}, {
"instanceName" : "JDBCProducer_02",
"library" : "streamsets-datacollector-jdbc-lib",
"stageName" : "com_streamsets_pipeline_stage_destination_jdbc_JdbcDTarget",
"stageVersion" : "6",
"configuration" : [ {
"name" : "schema",
"value" : ""
}, {
"name" : "tableNameTemplate",
"value" : "download"
}, {
"name" : "columnNames",
"value" : [ {
"paramValue" : "?",
"dataType" : "USE_COLUMN_TYPE",
"columnName" : "ip",
"field" : "/ip"
}, {
"paramValue" : "?",
"dataType" : "USE_COLUMN_TYPE",
"columnName" : "date",
"field" : "/timestamp"
}, {
"paramValue" : "?",
"dataType" : "USE_COLUMN_TYPE",
"columnName" : "file",
"field" : "/file"
} ]
}, {
"name" : "encloseTableName",
"value" : false
}, {
"name" : "changeLogFormat",
"value" : "NONE"
}, {
"name" : "defaultOperation",
"value" : "INSERT"
}, {
"name" : "unsupportedAction",
"value" : "DISCARD"
}, {
"name" : "useMultiRowInsert",
"value" : false
}, {
"name" : "maxPrepStmtParameters",
"value" : -1
}, {
"name" : "maxPrepStmtCache",
"value" : -1
}, {
"name" : "rollbackOnError",
"value" : false
}, {
"name" : "hikariConfigBean.connectionString",
"value" : "${JDBC_URI}?useSSL=false"
}, {
"name" : "hikariConfigBean.useCredentials",
"value" : true
}, {
"name" : "hikariConfigBean.username",
"value" : "${JDBC_USER}"
}, {
"name" : "hikariConfigBean.password",
"value" : "${JDBC_PASSWORD}"
}, {
"name" : "hikariConfigBean.driverProperties",
"value" : [ ]
}, {
"name" : "hikariConfigBean.driverClassName",
"value" : null
}, {
"name" : "hikariConfigBean.connectionTestQuery",
"value" : null
}, {
"name" : "hikariConfigBean.maximumPoolSize",
"value" : 1
}, {
"name" : "hikariConfigBean.minIdle",
"value" : 1
}, {
"name" : "hikariConfigBean.connectionTimeout",
"value" : "${30 * SECONDS}"
}, {
"name" : "hikariConfigBean.idleTimeout",
"value" : "${10 * MINUTES}"
}, {
"name" : "hikariConfigBean.maxLifetime",
"value" : "${30 * MINUTES}"
}, {
"name" : "hikariConfigBean.transactionIsolation",
"value" : "TRANSACTION_READ_COMMITTED"
}, {
"name" : "stageOnRecordError",
"value" : "TO_ERROR"
}, {
"name" : "stageRequiredFields",
"value" : [ ]
}, {
"name" : "stageRecordPreconditions",
"value" : [ ]
}, {
"name" : "hikariConfigBean.initialQuery",
"value" : null
} ],
"uiInfo" : {
"description" : "",
"label" : "download table",
"xPos" : 720,
"yPos" : 40,
"stageType" : "TARGET"
},
"inputLanes" : [ "StreamSelector_01OutputLane1499904663845" ],
"outputLanes" : [ ],
"eventLanes" : [ ],
"services" : [ ]
}, {
"instanceName" : "JDBCProducer_01",
"library" : "streamsets-datacollector-jdbc-lib",
"stageName" : "com_streamsets_pipeline_stage_destination_jdbc_JdbcDTarget",
"stageVersion" : "6",
"configuration" : [ {
"name" : "schema",
"value" : ""
}, {
"name" : "tableNameTemplate",
"value" : "log_entry"
}, {
"name" : "columnNames",
"value" : [ {
"paramValue" : "?",
"dataType" : "USE_COLUMN_TYPE",
"columnName" : "ip",
"field" : "/ip"
}, {
"paramValue" : "?",
"dataType" : "USE_COLUMN_TYPE",
"columnName" : "date",
"field" : "/timestamp"
}, {
"paramValue" : "?",
"dataType" : "USE_COLUMN_TYPE",
"columnName" : "verb",
"field" : "/verb"
}, {
"paramValue" : "?",
"dataType" : "USE_COLUMN_TYPE",
"columnName" : "file",
"field" : "/file"
}, {
"paramValue" : "?",
"dataType" : "USE_COLUMN_TYPE",
"columnName" : "code",
"field" : "/code",
"defaultValue" : "0"
}, {
"paramValue" : "?",
"dataType" : "USE_COLUMN_TYPE",
"columnName" : "size",
"field" : "/size",
"defaultValue" : "0"
} ]
}, {
"name" : "encloseTableName",
"value" : false
}, {
"name" : "changeLogFormat",
"value" : "NONE"
}, {
"name" : "defaultOperation",
"value" : "INSERT"
}, {
"name" : "unsupportedAction",
"value" : "DISCARD"
}, {
"name" : "useMultiRowInsert",
"value" : false
}, {
"name" : "maxPrepStmtParameters",
"value" : -1
}, {
"name" : "maxPrepStmtCache",
"value" : -1
}, {
"name" : "rollbackOnError",
"value" : false
}, {
"name" : "hikariConfigBean.connectionString",
"value" : "${JDBC_URI}?useSSL=false"
}, {
"name" : "hikariConfigBean.useCredentials",
"value" : true
}, {
"name" : "hikariConfigBean.username",
"value" : "${JDBC_USER}"
}, {
"name" : "hikariConfigBean.password",
"value" : "${JDBC_PASSWORD}"
}, {
"name" : "hikariConfigBean.driverProperties",
"value" : [ ]
}, {
"name" : "hikariConfigBean.driverClassName",
"value" : null
}, {
"name" : "hikariConfigBean.connectionTestQuery",
"value" : null
}, {
"name" : "hikariConfigBean.maximumPoolSize",
"value" : 1
}, {
"name" : "hikariConfigBean.minIdle",
"value" : 1
}, {
"name" : "hikariConfigBean.connectionTimeout",
"value" : "${30 * SECONDS}"
}, {
"name" : "hikariConfigBean.idleTimeout",
"value" : "${10 * MINUTES}"
}, {
"name" : "hikariConfigBean.maxLifetime",
"value" : "${30 * MINUTES}"
}, {
"name" : "hikariConfigBean.transactionIsolation",
"value" : "TRANSACTION_READ_COMMITTED"
}, {
"name" : "stageOnRecordError",
"value" : "TO_ERROR"
}, {
"name" : "stageRequiredFields",
"value" : [ ]
}, {
"name" : "stageRecordPreconditions",
"value" : [ ]
}, {
"name" : "hikariConfigBean.initialQuery",
"value" : null
} ],
"uiInfo" : {
"description" : "",
"label" : "log_entry table",
"xPos" : 720,
"yPos" : 170.0109405517578,
"stageType" : "TARGET"
},
"inputLanes" : [ "StreamSelector_01OutputLane1499903680944" ],
"outputLanes" : [ ],
"eventLanes" : [ ],
"services" : [ ]
}, {
"instanceName" : "Trash_01",
"library" : "streamsets-datacollector-basic-lib",
"stageName" : "com_streamsets_pipeline_stage_destination_devnull_NullDTarget",
"stageVersion" : "1",
"configuration" : [ ],
"uiInfo" : {
"description" : "",
"label" : "Trash 1",
"xPos" : 720,
"yPos" : 300,
"stageType" : "TARGET"
},
"inputLanes" : [ "StreamSelector_01OutputLane1499903546028" ],
"outputLanes" : [ ],
"eventLanes" : [ ],
"services" : [ ]
}, {
"instanceName" : "RecordDeduplicator_01",
"library" : "streamsets-datacollector-basic-lib",
"stageName" : "com_streamsets_pipeline_stage_processor_dedup_DeDupDProcessor",
"stageVersion" : "1",
"configuration" : [ {
"name" : "recordCountWindow",
"value" : 1000000
}, {
"name" : "timeWindowSecs",
"value" : 0
}, {
"name" : "compareFields",
"value" : "SPECIFIED_FIELDS"
}, {
"name" : "fieldsToCompare",
"value" : [ "/ip" ]
}, {
"name" : "stageRequiredFields",
"value" : [ ]
}, {
"name" : "stageRecordPreconditions",
"value" : [ ]
} ],
"uiInfo" : {
"outputStreamLabels" : [ "Unique Records", "Duplicate Records" ],
"yPos" : -94.90224838256836,
"stageType" : "PROCESSOR",
"description" : "",
"label" : "Deduplicate IPs",
"xPos" : 721.3631591796875
},
"inputLanes" : [ "StreamSelector_01OutputLane1499904663845" ],
"outputLanes" : [ "RecordDeduplicator_01OutputLane15015103760830", "RecordDeduplicator_01OutputLane15015103760831" ],
"eventLanes" : [ ],
"services" : [ ]
}, {
"instanceName" : "Trash_04",
"library" : "streamsets-datacollector-basic-lib",
"stageName" : "com_streamsets_pipeline_stage_destination_devnull_NullDTarget",
"stageVersion" : "1",
"configuration" : [ ],
"uiInfo" : {
"description" : "",
"label" : "Trash 4",
"xPos" : 935.0000610351562,
"yPos" : 45,
"stageType" : "TARGET"
},
"inputLanes" : [ "RecordDeduplicator_01OutputLane15015103760831" ],
"outputLanes" : [ ],
"eventLanes" : [ ],
"services" : [ ]
}, {
"instanceName" : "JDBCLookup_01",
"library" : "streamsets-datacollector-jdbc-lib",
"stageName" : "com_streamsets_pipeline_stage_processor_jdbclookup_JdbcLookupDProcessor",
"stageVersion" : "3",
"configuration" : [ {
"name" : "query",
"value" : "SELECT COUNT(*) AS count, COUNT(website) AS website\nFROM client \nWHERE ip = '${record:value('/ip')}'"
}, {
"name" : "columnMappings",
"value" : [ {
"dataType" : "USE_COLUMN_TYPE",
"columnName" : "count",
"field" : "/count",
"defaultValue" : ""
} ]
}, {
"name" : "multipleValuesBehavior",
"value" : "FIRST_ONLY"
}, {
"name" : "maxClobSize",
"value" : 1000
}, {
"name" : "maxBlobSize",
"value" : 1000
}, {
"name" : "hikariConfigBean.connectionString",
"value" : "${JDBC_URI}?useSSL=false"
}, {
"name" : "hikariConfigBean.useCredentials",
"value" : true
}, {
"name" : "hikariConfigBean.username",
"value" : "${JDBC_USER}"
}, {
"name" : "hikariConfigBean.password",
"value" : "${JDBC_PASSWORD}"
}, {
"name" : "hikariConfigBean.driverProperties",
"value" : [ ]
}, {
"name" : "hikariConfigBean.driverClassName",
"value" : null
}, {
"name" : "hikariConfigBean.connectionTestQuery",
"value" : null
}, {
"name" : "hikariConfigBean.maximumPoolSize",
"value" : 1
}, {
"name" : "hikariConfigBean.minIdle",
"value" : 1
}, {
"name" : "hikariConfigBean.connectionTimeout",
"value" : "${30 * SECONDS}"
}, {
"name" : "hikariConfigBean.idleTimeout",
"value" : "${10 * MINUTES}"
}, {
"name" : "hikariConfigBean.maxLifetime",
"value" : "${30 * MINUTES}"
}, {
"name" : "hikariConfigBean.autoCommit",
"value" : false
}, {
"name" : "hikariConfigBean.readOnly",
"value" : true
}, {
"name" : "hikariConfigBean.transactionIsolation",
"value" : "DEFAULT"
}, {
"name" : "cacheConfig.enabled",
"value" : false
}, {
"name" : "cacheConfig.maxSize",
"value" : -1
}, {
"name" : "cacheConfig.evictionPolicyType",
"value" : null
}, {
"name" : "cacheConfig.expirationTime",
"value" : 1
}, {
"name" : "cacheConfig.timeUnit",
"value" : "SECONDS"
}, {
"name" : "stageOnRecordError",
"value" : "TO_ERROR"
}, {
"name" : "stageRequiredFields",
"value" : [ ]
}, {
"name" : "stageRecordPreconditions",
"value" : [ ]
}, {
"name" : "hikariConfigBean.initialQuery",
"value" : null
}, {
"name" : "missingValuesBehavior",
"value" : "SEND_TO_ERROR"
}, {
"name" : "cacheConfig.retryOnCacheMiss",
"value" : false
} ],
"uiInfo" : {
"description" : "",
"label" : "client lookup",
"xPos" : 936.6666259765625,
"yPos" : -93.3333330154419,
"stageType" : "PROCESSOR"
},
"inputLanes" : [ "RecordDeduplicator_01OutputLane15015103760830" ],
"outputLanes" : [ "JDBCLookup_01OutputLane15000865896940" ],
"eventLanes" : [ ],
"services" : [ ]
}, {
"instanceName" : "StreamSelector_02",
"library" : "streamsets-datacollector-basic-lib",
"stageName" : "com_streamsets_pipeline_stage_processor_selector_SelectorDProcessor",
"stageVersion" : "1",
"configuration" : [ {
"name" : "lanePredicates",
"value" : [ {
"outputLane" : "StreamSelector_02OutputLane15000866189680",
"predicate" : "${record:value('/website') > 0}"
}, {
"outputLane" : "StreamSelector_02OutputLane15000866189681",
"predicate" : "default"
} ]
}, {
"name" : "stageOnRecordError",
"value" : "TO_ERROR"
}, {
"name" : "stageRequiredFields",
"value" : [ ]
}, {
"name" : "stageRecordPreconditions",
"value" : [ ]
} ],
"uiInfo" : {
"description" : "",
"label" : "website is set?",
"xPos" : 1143.86767578125,
"yPos" : -94.99998092651367,
"stageType" : "PROCESSOR"
},
"inputLanes" : [ "JDBCLookup_01OutputLane15000865896940" ],
"outputLanes" : [ "StreamSelector_02OutputLane15000866189680", "StreamSelector_02OutputLane15000866189681" ],
"eventLanes" : [ ],
"services" : [ ]
}, {
"instanceName" : "HTTPClient_01",
"library" : "streamsets-datacollector-basic-lib",
"stageName" : "com_streamsets_pipeline_stage_processor_http_HttpDProcessor",
"stageVersion" : "11",
"configuration" : [ {
"name" : "conf.dataFormatConfig.filePatternInArchive",
"value" : "*"
}, {
"name" : "conf.dataFormatConfig.charset",
"value" : "UTF-8"
}, {
"name" : "conf.dataFormatConfig.removeCtrlChars",
"value" : false
}, {
"name" : "conf.dataFormatConfig.textMaxLineLen",
"value" : 1024
}, {
"name" : "conf.dataFormatConfig.useCustomDelimiter",
"value" : false
}, {
"name" : "conf.dataFormatConfig.customDelimiter",
"value" : "\\r\\n"
}, {
"name" : "conf.dataFormatConfig.includeCustomDelimiterInTheText",
"value" : false
}, {
"name" : "conf.dataFormatConfig.jsonMaxObjectLen",
"value" : 4096
}, {
"name" : "conf.dataFormatConfig.csvFileFormat",
"value" : "CSV"
}, {
"name" : "conf.dataFormatConfig.csvHeader",
"value" : "NO_HEADER"
}, {
"name" : "conf.dataFormatConfig.csvMaxObjectLen",
"value" : 1024
}, {
"name" : "conf.dataFormatConfig.csvCustomDelimiter",
"value" : "|"
}, {
"name" : "conf.dataFormatConfig.csvCustomEscape",
"value" : "\\"
}, {
"name" : "conf.dataFormatConfig.csvCustomQuote",
"value" : "\""
}, {
"name" : "conf.dataFormatConfig.csvEnableComments",
"value" : false
}, {
"name" : "conf.dataFormatConfig.csvCommentMarker",
"value" : "#"
}, {
"name" : "conf.dataFormatConfig.csvIgnoreEmptyLines",
"value" : true
}, {
"name" : "conf.dataFormatConfig.csvRecordType",
"value" : "LIST_MAP"
}, {
"name" : "conf.dataFormatConfig.csvSkipStartLines",
"value" : 0
}, {
"name" : "conf.dataFormatConfig.parseNull",
"value" : false
}, {
"name" : "conf.dataFormatConfig.nullConstant",
"value" : "\\\\N"
}, {
"name" : "conf.dataFormatConfig.xmlRecordElement",
"value" : null
}, {
"name" : "conf.dataFormatConfig.includeFieldXpathAttributes",
"value" : false
}, {
"name" : "conf.dataFormatConfig.xPathNamespaceContext",
"value" : [ ]
}, {
"name" : "conf.dataFormatConfig.outputFieldAttributes",
"value" : false
}, {
"name" : "conf.dataFormatConfig.xmlMaxObjectLen",
"value" : 4096
}, {
"name" : "conf.dataFormatConfig.logMode",
"value" : "COMMON_LOG_FORMAT"
}, {
"name" : "conf.dataFormatConfig.logMaxObjectLen",
"value" : 1024
}, {
"name" : "conf.dataFormatConfig.retainOriginalLine",
"value" : false
}, {
"name" : "conf.dataFormatConfig.customLogFormat",
"value" : "%h %l %u %t \"%r\" %>s %b"
}, {
"name" : "conf.dataFormatConfig.regex",
"value" : "^(\\S+) (\\S+) (\\S+) \\[([\\w:/]+\\s[+\\-]\\d{4})\\] \"(\\S+) (\\S+) (\\S+)\" (\\d{3}) (\\d+)"
}, {
"name" : "conf.dataFormatConfig.fieldPathsToGroupName",
"value" : [ {
"fieldPath" : "/",
"group" : 1
} ]
}, {
"name" : "conf.dataFormatConfig.grokPatternDefinition",
"value" : null
}, {
"name" : "conf.dataFormatConfig.grokPattern",
"value" : "%{COMMONAPACHELOG}"
}, {
"name" : "conf.dataFormatConfig.onParseError",
"value" : "ERROR"
}, {
"name" : "conf.dataFormatConfig.maxStackTraceLines",
"value" : 50
}, {
"name" : "conf.dataFormatConfig.enableLog4jCustomLogFormat",
"value" : false
}, {
"name" : "conf.dataFormatConfig.log4jCustomLogFormat",
"value" : "%r [%t] %-5p %c %x - %m%n"
}, {
"name" : "conf.dataFormatConfig.avroSchemaSource",
"value" : null
}, {
"name" : "conf.dataFormatConfig.avroSchema",
"value" : null
}, {
"name" : "conf.dataFormatConfig.schemaRegistryUrls",
"value" : [ ]
}, {
"name" : "conf.dataFormatConfig.schemaLookupMode",
"value" : "SUBJECT"
}, {
"name" : "conf.dataFormatConfig.subject",
"value" : null
}, {
"name" : "conf.dataFormatConfig.schemaId",
"value" : null
}, {
"name" : "conf.dataFormatConfig.protoDescriptorFile",
"value" : null
}, {
"name" : "conf.dataFormatConfig.messageType",
"value" : null
}, {
"name" : "conf.dataFormatConfig.isDelimited",
"value" : true
}, {
"name" : "conf.dataFormatConfig.binaryMaxObjectLen",
"value" : 1024
}, {
"name" : "conf.dataFormatConfig.datagramMode",
"value" : "SYSLOG"
}, {
"name" : "conf.dataFormatConfig.typesDbPath",
"value" : null
}, {
"name" : "conf.dataFormatConfig.convertTime",
"value" : false
}, {
"name" : "conf.dataFormatConfig.excludeInterval",
"value" : true
}, {
"name" : "conf.dataFormatConfig.authFilePath",
"value" : null
}, {
"name" : "conf.dataFormatConfig.wholeFileMaxObjectLen",
"value" : 8192
}, {
"name" : "conf.dataFormatConfig.rateLimit",
"value" : "-1"
}, {
"name" : "conf.dataFormatConfig.verifyChecksum",
"value" : false
}, {
"name" : "conf.outputField",
"value" : "/lookup"
}, {
"name" : "conf.headerOutputLocation",
"value" : "HEADER"
}, {
"name" : "conf.headerAttributePrefix",
"value" : null
}, {
"name" : "conf.headerOutputField",
"value" : null
}, {
"name" : "conf.dataFormat",
"value" : "JSON"
}, {
"name" : "conf.resourceUrl",
"value" : "https://api.kickfire.com/v2/company:(all)?ip=${record:value('/ip')}&key=${KICKFIRE_KEY}"
}, {
"name" : "conf.headers",
"value" : [ ]
}, {
"name" : "conf.httpMethod",
"value" : "GET"
}, {
"name" : "conf.methodExpression",
"value" : null
}, {
"name" : "conf.requestBody",
"value" : null
}, {
"name" : "conf.defaultRequestContentType",
"value" : "application/json"
}, {
"name" : "conf.client.transferEncoding",
"value" : "CHUNKED"
}, {
"name" : "conf.client.httpCompression",
"value" : "NONE"
}, {
"name" : "conf.client.connectTimeoutMillis",
"value" : 0
}, {
"name" : "conf.client.readTimeoutMillis",
"value" : 0
}, {
"name" : "conf.client.numThreads",
"value" : 1
}, {
"name" : "conf.client.authType",
"value" : "NONE"
}, {
"name" : "conf.client.useOAuth2",
"value" : false
}, {
"name" : "conf.client.oauth.consumerKey",
"value" : ""
}, {
"name" : "conf.client.oauth.consumerSecret",
"value" : ""
}, {
"name" : "conf.client.oauth.token",
"value" : ""
}, {
"name" : "conf.client.oauth.tokenSecret",
"value" : ""
}, {
"name" : "conf.client.oauth2.credentialsGrantType",
"value" : null
}, {
"name" : "conf.client.oauth2.tokenUrl",
"value" : null
}, {
"name" : "conf.client.oauth2.clientId",
"value" : ""
}, {
"name" : "conf.client.oauth2.clientSecret",
"value" : ""
}, {
"name" : "conf.client.oauth2.username",
"value" : ""
}, {
"name" : "conf.client.oauth2.password",
"value" : ""
}, {
"name" : "conf.client.oauth2.resourceOwnerClientId",
"value" : ""
}, {
"name" : "conf.client.oauth2.resourceOwnerClientSecret",
"value" : ""
}, {
"name" : "conf.client.oauth2.algorithm",
"value" : "NONE"
}, {
"name" : "conf.client.oauth2.key",
"value" : ""
}, {
"name" : "conf.client.oauth2.jwtClaims",
"value" : null
}, {
"name" : "conf.client.oauth2.transferEncoding",
"value" : "BUFFERED"
}, {
"name" : "conf.client.oauth2.additionalValues",
"value" : [ ]
}, {
"name" : "conf.client.basicAuth.username",
"value" : ""
}, {
"name" : "conf.client.basicAuth.password",
"value" : ""
}, {
"name" : "conf.client.useProxy",
"value" : false
}, {
"name" : "conf.client.proxy.uri",
"value" : null
}, {
"name" : "conf.client.proxy.username",
"value" : ""
}, {
"name" : "conf.client.proxy.password",
"value" : ""
}, {
"name" : "conf.client.tlsConfig.tlsEnabled",
"value" : false
}, {
"name" : "conf.client.tlsConfig.trustStoreFilePath",
"value" : null
}, {
"name" : "conf.client.tlsConfig.trustStoreType",
"value" : "JKS"
}, {
"name" : "conf.client.tlsConfig.trustStorePassword",
"value" : ""
}, {
"name" : "conf.client.tlsConfig.trustStoreAlgorithm",
"value" : "SunX509"
}, {
"name" : "conf.client.tlsConfig.useDefaultProtocols",
"value" : true
}, {
"name" : "conf.client.tlsConfig.protocols",
"value" : [ ]
}, {
"name" : "conf.client.tlsConfig.useDefaultCiperSuites",
"value" : true
}, {
"name" : "conf.client.tlsConfig.cipherSuites",
"value" : [ ]
}, {
"name" : "conf.maxRequestCompletionSecs",
"value" : 60
}, {
"name" : "stageOnRecordError",
"value" : "STOP_PIPELINE"
}, {
"name" : "stageRequiredFields",
"value" : [ ]
}, {
"name" : "stageRecordPreconditions",
"value" : [ ]
}, {
"name" : "conf.rateLimit",
"value" : 3000
}, {
"name" : "conf.client.requestLoggingConfig.enableRequestLogging",
"value" : false
}, {
"name" : "conf.client.requestLoggingConfig.logLevel",
"value" : "FINE"
}, {
"name" : "conf.client.requestLoggingConfig.verbosity",
"value" : "HEADERS_ONLY"
}, {
"name" : "conf.client.requestLoggingConfig.maxEntitySize",
"value" : 0
}, {
"name" : "conf.dataFormatConfig.csvAllowExtraColumns",
"value" : false
}, {
"name" : "conf.dataFormatConfig.csvExtraColumnPrefix",
"value" : "_extra_"
}, {
"name" : "conf.dataFormatConfig.netflowOutputValuesMode",
"value" : "RAW_AND_INTERPRETED"
}, {
"name" : "conf.dataFormatConfig.maxTemplateCacheSize",
"value" : -1
}, {
"name" : "conf.dataFormatConfig.templateCacheTimeoutMs",
"value" : -1
}, {
"name" : "conf.dataFormatConfig.netflowOutputValuesModeDatagram",
"value" : "RAW_AND_INTERPRETED"
}, {
"name" : "conf.dataFormatConfig.maxTemplateCacheSizeDatagram",
"value" : -1
}, {
"name" : "conf.dataFormatConfig.templateCacheTimeoutMsDatagram",
"value" : -1
}, {
"name" : "conf.client.tlsConfig.keyStoreFilePath",
"value" : null
}, {
"name" : "conf.client.tlsConfig.keyStoreType",
"value" : "JKS"
}, {
"name" : "conf.client.tlsConfig.keyStorePassword",
"value" : ""
}, {
"name" : "conf.client.tlsConfig.keyStoreAlgorithm",
"value" : "SunX509"
}, {
"name" : "conf.dataFormatConfig.compression",
"value" : "NONE"
}, {
"name" : "conf.dataFormatConfig.excelHeader",
"value" : null
}, {
"name" : "conf.dataFormatConfig.multiCharacterFieldDelimiter",
"value" : "||"
}, {
"name" : "conf.dataFormatConfig.multiCharacterLineDelimiter",
"value" : "${str:unescapeJava('\\\\n')}"
} ],
"uiInfo" : {
"description" : "",
"label" : "Kickfire Company Lookup",
"xPos" : 1364.79052734375,
"yPos" : 50.00000762939453,
"stageType" : "PROCESSOR"
},
"inputLanes" : [ "StreamSelector_02OutputLane15000866189681" ],
"outputLanes" : [ "HTTPClient_01OutputLane15000866409770" ],
"eventLanes" : [ ],
"services" : [ ]
}, {
"instanceName" : "Trash_02",
"library" : "streamsets-datacollector-basic-lib",
"stageName" : "com_streamsets_pipeline_stage_destination_devnull_NullDTarget",
"stageVersion" : "1",
"configuration" : [ ],
"uiInfo" : {
"description" : "",
"label" : "Trash 2",
"xPos" : 1363.3333740234375,
"yPos" : -91.66665267944336,
"stageType" : "TARGET"
},
"inputLanes" : [ "StreamSelector_02OutputLane15000866189680" ],
"outputLanes" : [ ],
"eventLanes" : [ ],
"services" : [ ]
}, {
"instanceName" : "StreamSelector_04",
"library" : "streamsets-datacollector-basic-lib",
"stageName" : "com_streamsets_pipeline_stage_processor_selector_SelectorDProcessor",
"stageVersion" : "1",
"configuration" : [ {
"name" : "lanePredicates",
"value" : [ {
"outputLane" : "StreamSelector_04OutputLane1500087032682",
"predicate" : "${record:value('/lookup/status') == 'success'}"
}, {
"outputLane" : "StreamSelector_04OutputLane1500087006721",
"predicate" : "default"
} ]
}, {
"name" : "stageOnRecordError",
"value" : "TO_ERROR"
}, {
"name" : "stageRequiredFields",
"value" : [ ]
}, {
"name" : "stageRecordPreconditions",
"value" : [ ]
} ],
"uiInfo" : {
"description" : "",
"label" : "Kickfire success?",
"xPos" : 1565,
"yPos" : 51.66667938232422,
"stageType" : "PROCESSOR"
},
"inputLanes" : [ "HTTPClient_01OutputLane15000866409770" ],
"outputLanes" : [ "StreamSelector_04OutputLane1500087032682", "StreamSelector_04OutputLane1500087006721" ],
"eventLanes" : [ ],
"services" : [ ]
}, {
"instanceName" : "ToError_01",
"library" : "streamsets-datacollector-basic-lib",
"stageName" : "com_streamsets_pipeline_stage_destination_toerror_ToErrorDTarget",
"stageVersion" : "1",
"configuration" : [ ],
"uiInfo" : {
"description" : "",
"label" : "To Error 1",
"xPos" : 1818.333251953125,
"yPos" : 178.33334350585938,
"stageType" : "TARGET"
},
"inputLanes" : [ "StreamSelector_04OutputLane1500087006721" ],
"outputLanes" : [ ],
"eventLanes" : [ ],
"services" : [ ]
}, {
"instanceName" : "RecordDeduplicator_02",
"library" : "streamsets-datacollector-basic-lib",
"stageName" : "com_streamsets_pipeline_stage_processor_dedup_DeDupDProcessor",
"stageVersion" : "1",
"configuration" : [ {
"name" : "recordCountWindow",
"value" : 1000000
}, {
"name" : "timeWindowSecs",
"value" : 0
}, {
"name" : "compareFields",
"value" : "SPECIFIED_FIELDS"
}, {
"name" : "fieldsToCompare",
"value" : [ "/lookup/data[0]/name" ]
}, {
"name" : "stageRequiredFields",
"value" : [ "/lookup/data[0]/name" ]
}, {
"name" : "stageRecordPreconditions",
"value" : [ ]
} ],
"uiInfo" : {
"outputStreamLabels" : [ "Unique Records", "Duplicate Records" ],
"yPos" : 46.66667175292969,
"stageType" : "PROCESSOR",
"description" : "",
"label" : "Deduplicate Company Names",
"xPos" : 1811.666748046875
},
"inputLanes" : [ "StreamSelector_04OutputLane1500087032682" ],
"outputLanes" : [ "RecordDeduplicator_02OutputLane15015108315570", "RecordDeduplicator_02OutputLane15015108315571" ],
"eventLanes" : [ ],
"services" : [ ]
}, {
"instanceName" : "JDBCLookup_02",
"library" : "streamsets-datacollector-jdbc-lib",
"stageName" : "com_streamsets_pipeline_stage_processor_jdbclookup_JdbcLookupDProcessor",
"stageVersion" : "3",
"configuration" : [ {
"name" : "query",
"value" : "SELECT COUNT(*) as count FROM org_company WHERE org = '${record:value('/lookup/data[0]/name')}'"
}, {
"name" : "columnMappings",
"value" : [ {
"dataType" : "USE_COLUMN_TYPE",
"columnName" : "count",
"field" : "/count",
"defaultValue" : ""
} ]
}, {
"name" : "multipleValuesBehavior",
"value" : "FIRST_ONLY"
}, {
"name" : "maxClobSize",
"value" : 1000
}, {
"name" : "maxBlobSize",
"value" : 1000
}, {
"name" : "hikariConfigBean.connectionString",
"value" : "${JDBC_URI}?useSSL=false"
}, {
"name" : "hikariConfigBean.useCredentials",
"value" : true
}, {
"name" : "hikariConfigBean.username",
"value" : "${JDBC_USER}"
}, {
"name" : "hikariConfigBean.password",
"value" : "${JDBC_PASSWORD}"
}, {
"name" : "hikariConfigBean.driverProperties",
"value" : [ ]
}, {
"name" : "hikariConfigBean.driverClassName",
"value" : null
}, {
"name" : "hikariConfigBean.connectionTestQuery",
"value" : null
}, {
"name" : "hikariConfigBean.maximumPoolSize",
"value" : 1
}, {
"name" : "hikariConfigBean.minIdle",
"value" : 1
}, {
"name" : "hikariConfigBean.connectionTimeout",
"value" : "${30 * SECONDS}"
}, {
"name" : "hikariConfigBean.idleTimeout",
"value" : "${10 * MINUTES}"
}, {
"name" : "hikariConfigBean.maxLifetime",
"value" : "${30 * MINUTES}"
}, {
"name" : "hikariConfigBean.autoCommit",
"value" : false
}, {
"name" : "hikariConfigBean.readOnly",
"value" : true
}, {
"name" : "hikariConfigBean.transactionIsolation",
"value" : "DEFAULT"
}, {
"name" : "cacheConfig.enabled",
"value" : false
}, {
"name" : "cacheConfig.maxSize",
"value" : -1
}, {
"name" : "cacheConfig.evictionPolicyType",
"value" : null
}, {
"name" : "cacheConfig.expirationTime",
"value" : 1
}, {
"name" : "cacheConfig.timeUnit",
"value" : "SECONDS"
}, {
"name" : "stageOnRecordError",
"value" : "TO_ERROR"
}, {
"name" : "stageRequiredFields",
"value" : [ ]
}, {
"name" : "stageRecordPreconditions",
"value" : [ ]
}, {
"name" : "hikariConfigBean.initialQuery",
"value" : null
}, {
"name" : "missingValuesBehavior",
"value" : "SEND_TO_ERROR"
}, {
"name" : "cacheConfig.retryOnCacheMiss",
"value" : false
} ],
"uiInfo" : {
"description" : "",
"label" : "org/company lookup",
"xPos" : 2041.666748046875,
"yPos" : 50.00001525878906,
"stageType" : "PROCESSOR"
},
"inputLanes" : [ "RecordDeduplicator_02OutputLane15015108315570" ],
"outputLanes" : [ "JDBCLookup_02OutputLane15000867578550" ],
"eventLanes" : [ ],
"services" : [ ]
}, {
"instanceName" : "StreamSelector_03",
"library" : "streamsets-datacollector-basic-lib",
"stageName" : "com_streamsets_pipeline_stage_processor_selector_SelectorDProcessor",
"stageVersion" : "1",
"configuration" : [ {
"name" : "lanePredicates",
"value" : [ {
"outputLane" : "StreamSelector_03OutputLane15000868308540",
"predicate" : "${record:value('/count') > 0}"
}, {
"outputLane" : "StreamSelector_03OutputLane15000868308541",
"predicate" : "default"
} ]
}, {
"name" : "stageOnRecordError",
"value" : "TO_ERROR"
}, {
"name" : "stageRequiredFields",
"value" : [ ]
}, {
"name" : "stageRecordPreconditions",
"value" : [ ]
} ],
"uiInfo" : {
"description" : "",
"label" : "org/company exists?",
"xPos" : 2246.666748046875,
"yPos" : 46.66667175292969,
"stageType" : "PROCESSOR"
},
"inputLanes" : [ "JDBCLookup_02OutputLane15000867578550" ],
"outputLanes" : [ "StreamSelector_03OutputLane15000868308540", "StreamSelector_03OutputLane15000868308541" ],
"eventLanes" : [ ],
"services" : [ ]
}, {
"instanceName" : "Trash_03",
"library" : "streamsets-datacollector-basic-lib",
"stageName" : "com_streamsets_pipeline_stage_destination_devnull_NullDTarget",
"stageVersion" : "1",
"configuration" : [ ],
"uiInfo" : {
"description" : "",
"label" : "Trash 3",
"xPos" : 2461.666748046875,
"yPos" : 45.00001525878906,
"stageType" : "TARGET"
},
"inputLanes" : [ "StreamSelector_03OutputLane15000868308540" ],
"outputLanes" : [ ],
"eventLanes" : [ ],
"services" : [ ]
}, {
"instanceName" : "JDBCProducer_04",
"library" : "streamsets-datacollector-jdbc-lib",
"stageName" : "com_streamsets_pipeline_stage_destination_jdbc_JdbcDTarget",
"stageVersion" : "6",
"configuration" : [ {
"name" : "schema",
"value" : ""
}, {
"name" : "tableNameTemplate",
"value" : "org_company"
}, {
"name" : "columnNames",
"value" : [ {
"paramValue" : "?",
"dataType" : "USE_COLUMN_TYPE",
"columnName" : "company",
"field" : "/lookup/data[0]/name"
}, {
"paramValue" : "?",
"dataType" : "USE_COLUMN_TYPE",
"columnName" : "org",
"field" : "/lookup/data[0]/name"
} ]
}, {
"name" : "encloseTableName",
"value" : false
}, {
"name" : "changeLogFormat",
"value" : "NONE"
}, {
"name" : "defaultOperation",
"value" : "INSERT"
}, {
"name" : "unsupportedAction",
"value" : "DISCARD"
}, {
"name" : "useMultiRowInsert",
"value" : false
}, {
"name" : "maxPrepStmtParameters",
"value" : -1
}, {
"name" : "maxPrepStmtCache",
"value" : -1
}, {
"name" : "rollbackOnError",
"value" : false
}, {
"name" : "hikariConfigBean.connectionString",
"value" : "${JDBC_URI}?useSSL=false"
}, {
"name" : "hikariConfigBean.useCredentials",
"value" : true
}, {
"name" : "hikariConfigBean.username",
"value" : "${JDBC_USER}"
}, {
"name" : "hikariConfigBean.password",
"value" : "${JDBC_PASSWORD}"
}, {
"name" : "hikariConfigBean.driverProperties",
"value" : [ ]
}, {
"name" : "hikariConfigBean.driverClassName",
"value" : null
}, {
"name" : "hikariConfigBean.connectionTestQuery",
"value" : null
}, {
"name" : "hikariConfigBean.maximumPoolSize",
"value" : 1
}, {
"name" : "hikariConfigBean.minIdle",
"value" : 1
}, {
"name" : "hikariConfigBean.connectionTimeout",
"value" : "${30 * SECONDS}"
}, {
"name" : "hikariConfigBean.idleTimeout",
"value" : "${10 * MINUTES}"
}, {
"name" : "hikariConfigBean.maxLifetime",
"value" : "${30 * MINUTES}"
}, {
"name" : "hikariConfigBean.transactionIsolation",
"value" : "DEFAULT"
}, {
"name" : "stageOnRecordError",
"value" : "TO_ERROR"
}, {
"name" : "stageRequiredFields",
"value" : [ ]
}, {
"name" : "stageRecordPreconditions",
"value" : [ ]
}, {
"name" : "hikariConfigBean.initialQuery",
"value" : null
} ],
"uiInfo" : {
"description" : "",
"label" : "org_company table",
"xPos" : 2461.666748046875,
"yPos" : 181.6666717529297,
"stageType" : "TARGET"
},
"inputLanes" : [ "StreamSelector_03OutputLane15000868308541" ],
"outputLanes" : [ ],
"eventLanes" : [ ],
"services" : [ ]
}, {
"instanceName" : "Trash_05",
"library" : "streamsets-datacollector-basic-lib",
"stageName" : "com_streamsets_pipeline_stage_destination_devnull_NullDTarget",
"stageVersion" : "1",
"configuration" : [ ],
"uiInfo" : {
"description" : "",
"label" : "Trash 5",
"xPos" : 2045,
"yPos" : 186.6666717529297,
"stageType" : "TARGET"
},
"inputLanes" : [ "RecordDeduplicator_02OutputLane15015108315571" ],
"outputLanes" : [ ],
"eventLanes" : [ ],
"services" : [ ]
}, {
"instanceName" : "HTTPClient_02",
"library" : "streamsets-datacollector-basic-lib",
"stageName" : "com_streamsets_pipeline_stage_processor_http_HttpDProcessor",
"stageVersion" : "11",
"configuration" : [ {
"name" : "conf.dataFormatConfig.compression",
"value" : "NONE"
}, {
"name" : "conf.dataFormatConfig.filePatternInArchive",
"value" : "*"
}, {
"name" : "conf.dataFormatConfig.charset",
"value" : "UTF-8"
}, {
"name" : "conf.dataFormatConfig.removeCtrlChars",
"value" : false
}, {
"name" : "conf.dataFormatConfig.textMaxLineLen",
"value" : 1024
}, {
"name" : "conf.dataFormatConfig.useCustomDelimiter",
"value" : false
}, {
"name" : "conf.dataFormatConfig.customDelimiter",
"value" : "\\r\\n"
}, {
"name" : "conf.dataFormatConfig.includeCustomDelimiterInTheText",
"value" : false
}, {
"name" : "conf.dataFormatConfig.jsonMaxObjectLen",
"value" : 4096
}, {
"name" : "conf.dataFormatConfig.csvFileFormat",
"value" : "CSV"
}, {
"name" : "conf.dataFormatConfig.csvHeader",
"value" : "NO_HEADER"
}, {
"name" : "conf.dataFormatConfig.csvAllowExtraColumns",
"value" : false
}, {
"name" : "conf.dataFormatConfig.csvExtraColumnPrefix",
"value" : "_extra_"
}, {
"name" : "conf.dataFormatConfig.csvMaxObjectLen",
"value" : 1024
}, {
"name" : "conf.dataFormatConfig.csvCustomDelimiter",
"value" : "|"
}, {
"name" : "conf.dataFormatConfig.csvCustomEscape",
"value" : "\\"
}, {
"name" : "conf.dataFormatConfig.csvCustomQuote",
"value" : "\""
}, {
"name" : "conf.dataFormatConfig.csvEnableComments",
"value" : false
}, {
"name" : "conf.dataFormatConfig.csvCommentMarker",
"value" : "#"
}, {
"name" : "conf.dataFormatConfig.csvIgnoreEmptyLines",
"value" : true
}, {
"name" : "conf.dataFormatConfig.csvRecordType",
"value" : "LIST_MAP"
}, {
"name" : "conf.dataFormatConfig.csvSkipStartLines",
"value" : 0
}, {
"name" : "conf.dataFormatConfig.parseNull",
"value" : false
}, {
"name" : "conf.dataFormatConfig.nullConstant",
"value" : "\\\\N"
}, {
"name" : "conf.dataFormatConfig.xmlRecordElement",
"value" : null
}, {
"name" : "conf.dataFormatConfig.includeFieldXpathAttributes",
"value" : false
}, {
"name" : "conf.dataFormatConfig.xPathNamespaceContext",
"value" : [ ]
}, {
"name" : "conf.dataFormatConfig.outputFieldAttributes",
"value" : false
}, {
"name" : "conf.dataFormatConfig.xmlMaxObjectLen",
"value" : 4096
}, {
"name" : "conf.dataFormatConfig.logMode",
"value" : "COMMON_LOG_FORMAT"
}, {
"name" : "conf.dataFormatConfig.logMaxObjectLen",
"value" : 1024
}, {
"name" : "conf.dataFormatConfig.retainOriginalLine",
"value" : false
}, {
"name" : "conf.dataFormatConfig.customLogFormat",
"value" : "%h %l %u %t \"%r\" %>s %b"
}, {
"name" : "conf.dataFormatConfig.regex",
"value" : "^(\\S+) (\\S+) (\\S+) \\[([\\w:/]+\\s[+\\-]\\d{4})\\] \"(\\S+) (\\S+) (\\S+)\" (\\d{3}) (\\d+)"
}, {
"name" : "conf.dataFormatConfig.fieldPathsToGroupName",
"value" : [ {
"fieldPath" : "/",
"group" : 1
} ]
}, {
"name" : "conf.dataFormatConfig.grokPatternDefinition",
"value" : null
}, {
"name" : "conf.dataFormatConfig.grokPattern",
"value" : "%{COMMONAPACHELOG}"
}, {
"name" : "conf.dataFormatConfig.onParseError",
"value" : "ERROR"
}, {
"name" : "conf.dataFormatConfig.maxStackTraceLines",
"value" : 50
}, {
"name" : "conf.dataFormatConfig.enableLog4jCustomLogFormat",
"value" : false
}, {
"name" : "conf.dataFormatConfig.log4jCustomLogFormat",
"value" : "%r [%t] %-5p %c %x - %m%n"
}, {
"name" : "conf.dataFormatConfig.avroSchemaSource",
"value" : null
}, {
"name" : "conf.dataFormatConfig.avroSchema",
"value" : null
}, {
"name" : "conf.dataFormatConfig.schemaRegistryUrls",
"value" : [ ]
}, {
"name" : "conf.dataFormatConfig.schemaLookupMode",
"value" : "SUBJECT"
}, {
"name" : "conf.dataFormatConfig.subject",
"value" : null
}, {
"name" : "conf.dataFormatConfig.schemaId",
"value" : null
}, {
"name" : "conf.dataFormatConfig.protoDescriptorFile",
"value" : null
}, {
"name" : "conf.dataFormatConfig.messageType",
"value" : null
}, {
"name" : "conf.dataFormatConfig.isDelimited",
"value" : true
}, {
"name" : "conf.dataFormatConfig.binaryMaxObjectLen",
"value" : 1024
}, {
"name" : "conf.dataFormatConfig.datagramMode",
"value" : "SYSLOG"
}, {
"name" : "conf.dataFormatConfig.typesDbPath",
"value" : null
}, {
"name" : "conf.dataFormatConfig.convertTime",
"value" : false
}, {
"name" : "conf.dataFormatConfig.excludeInterval",
"value" : true
}, {
"name" : "conf.dataFormatConfig.authFilePath",
"value" : null
}, {
"name" : "conf.dataFormatConfig.netflowOutputValuesMode",
"value" : "RAW_AND_INTERPRETED"
}, {
"name" : "conf.dataFormatConfig.maxTemplateCacheSize",
"value" : -1
}, {
"name" : "conf.dataFormatConfig.templateCacheTimeoutMs",
"value" : -1
}, {
"name" : "conf.dataFormatConfig.netflowOutputValuesModeDatagram",
"value" : "RAW_AND_INTERPRETED"
}, {
"name" : "conf.dataFormatConfig.maxTemplateCacheSizeDatagram",
"value" : -1
}, {
"name" : "conf.dataFormatConfig.templateCacheTimeoutMsDatagram",
"value" : -1
}, {
"name" : "conf.dataFormatConfig.wholeFileMaxObjectLen",
"value" : 8192
}, {
"name" : "conf.dataFormatConfig.rateLimit",
"value" : "-1"
}, {
"name" : "conf.dataFormatConfig.verifyChecksum",
"value" : false
}, {
"name" : "conf.outputField",
"value" : "/location"
}, {
"name" : "conf.headerOutputLocation",
"value" : "HEADER"
}, {
"name" : "conf.headerAttributePrefix",
"value" : null
}, {
"name" : "conf.headerOutputField",
"value" : null
}, {
"name" : "conf.dataFormat",
"value" : "JSON"
}, {
"name" : "conf.resourceUrl",
"value" : "https://api.kickfire.com/v2/ip2geo?ip=${record:value('/ip')}&key=fb6d3463b00f3276"
}, {
"name" : "conf.headers",
"value" : [ ]
}, {
"name" : "conf.httpMethod",
"value" : "GET"
}, {
"name" : "conf.methodExpression",
"value" : null
}, {
"name" : "conf.requestBody",
"value" : null
}, {
"name" : "conf.defaultRequestContentType",
"value" : "application/json"
}, {
"name" : "conf.client.transferEncoding",
"value" : "CHUNKED"
}, {
"name" : "conf.client.httpCompression",
"value" : "NONE"
}, {
"name" : "conf.client.connectTimeoutMillis",
"value" : 0
}, {
"name" : "conf.client.readTimeoutMillis",
"value" : 0
}, {
"name" : "conf.client.numThreads",
"value" : 1
}, {
"name" : "conf.client.authType",
"value" : "NONE"
}, {
"name" : "conf.client.useOAuth2",
"value" : false
}, {
"name" : "conf.client.oauth.consumerKey",
"value" : ""
}, {
"name" : "conf.client.oauth.consumerSecret",
"value" : ""
}, {
"name" : "conf.client.oauth.token",
"value" : ""
}, {
"name" : "conf.client.oauth.tokenSecret",
"value" : ""
}, {
"name" : "conf.client.oauth2.credentialsGrantType",
"value" : null
}, {
"name" : "conf.client.oauth2.tokenUrl",
"value" : null
}, {
"name" : "conf.client.oauth2.clientId",
"value" : ""
}, {
"name" : "conf.client.oauth2.clientSecret",
"value" : ""
}, {
"name" : "conf.client.oauth2.username",
"value" : ""
}, {
"name" : "conf.client.oauth2.password",
"value" : ""
}, {
"name" : "conf.client.oauth2.resourceOwnerClientId",
"value" : ""
}, {
"name" : "conf.client.oauth2.resourceOwnerClientSecret",
"value" : ""
}, {
"name" : "conf.client.oauth2.algorithm",
"value" : "NONE"
}, {
"name" : "conf.client.oauth2.key",
"value" : ""
}, {
"name" : "conf.client.oauth2.jwtClaims",
"value" : null
}, {
"name" : "conf.client.oauth2.transferEncoding",
"value" : "BUFFERED"
}, {
"name" : "conf.client.oauth2.additionalValues",
"value" : [ { } ]
}, {
"name" : "conf.client.basicAuth.username",
"value" : ""
}, {
"name" : "conf.client.basicAuth.password",
"value" : ""
}, {
"name" : "conf.client.useProxy",
"value" : false
}, {
"name" : "conf.client.proxy.uri",
"value" : null
}, {
"name" : "conf.client.proxy.username",
"value" : ""
}, {
"name" : "conf.client.proxy.password",
"value" : ""
}, {
"name" : "conf.client.tlsConfig.tlsEnabled",
"value" : false
}, {
"name" : "conf.client.tlsConfig.keyStoreFilePath",
"value" : null
}, {
"name" : "conf.client.tlsConfig.keyStoreType",
"value" : "JKS"
}, {
"name" : "conf.client.tlsConfig.keyStorePassword",
"value" : ""
}, {
"name" : "conf.client.tlsConfig.keyStoreAlgorithm",
"value" : "SunX509"
}, {
"name" : "conf.client.tlsConfig.trustStoreFilePath",
"value" : null
}, {
"name" : "conf.client.tlsConfig.trustStoreType",
"value" : "JKS"
}, {
"name" : "conf.client.tlsConfig.trustStorePassword",
"value" : ""
}, {
"name" : "conf.client.tlsConfig.trustStoreAlgorithm",
"value" : "SunX509"
}, {
"name" : "conf.client.tlsConfig.useDefaultProtocols",
"value" : true
}, {
"name" : "conf.client.tlsConfig.protocols",
"value" : [ ]
}, {
"name" : "conf.client.tlsConfig.useDefaultCiperSuites",
"value" : true
}, {
"name" : "conf.client.tlsConfig.cipherSuites",
"value" : [ ]
}, {
"name" : "conf.client.requestLoggingConfig.enableRequestLogging",
"value" : false
}, {
"name" : "conf.client.requestLoggingConfig.logLevel",
"value" : "FINE"
}, {
"name" : "conf.client.requestLoggingConfig.verbosity",
"value" : "HEADERS_ONLY"
}, {
"name" : "conf.client.requestLoggingConfig.maxEntitySize",
"value" : 0
}, {
"name" : "conf.rateLimit",
"value" : 3000
}, {
"name" : "conf.maxRequestCompletionSecs",
"value" : 60
}, {
"name" : "stageOnRecordError",
"value" : "TO_ERROR"
}, {
"name" : "stageRequiredFields",
"value" : [ ]
}, {
"name" : "stageRecordPreconditions",
"value" : [ ]
}, {
"name" : "conf.dataFormatConfig.excelHeader",
"value" : null
}, {
"name" : "conf.dataFormatConfig.multiCharacterFieldDelimiter",
"value" : "||"
}, {
"name" : "conf.dataFormatConfig.multiCharacterLineDelimiter",
"value" : "${str:unescapeJava('\\\\n')}"
} ],
"uiInfo" : {
"outputStreamLabels" : null,
"yPos" : -84.0590705871582,
"stageType" : "PROCESSOR",
"rawSource" : null,
"description" : "",
"label" : "Kickfire Location Lookup",
"xPos" : 1811.983154296875
},
"inputLanes" : [ "StreamSelector_04OutputLane1500087032682" ],
"outputLanes" : [ "HTTPClient_02OutputLane15269275492590" ],
"eventLanes" : [ ],
"services" : [ ]
}, {
"instanceName" : "ExpressionEvaluator_01",
"library" : "streamsets-datacollector-basic-lib",
"stageName" : "com_streamsets_pipeline_stage_processor_expression_ExpressionDProcessor",
"stageVersion" : "2",
"configuration" : [ {
"name" : "expressionProcessorConfigs",
"value" : [ {
"fieldToSet" : "/location/data/latitude",
"expression" : "${str:length(record:value('/location/data/latitude')) == 0 ? '0' : record:value('/location/data/latitude')}"
}, {
"fieldToSet" : "/location/data/longitude",
"expression" : "${str:length(record:value('/location/data/longitude')) == 0 ? '0' : record:value('/location/data/longitude')}"
}, {
"fieldToSet" : "/lookup/data[0]/naicsCode",
"expression" : "${str:length(record:value('/lookup/data[0]/naicsCode')) == 0 ? '0' : record:value('/lookup/data[0]/naicsCode')}"
} ]
}, {
"name" : "headerAttributeConfigs",
"value" : [ {
"attributeToSet" : "sdc.operation.type",
"headerAttributeExpression" : "${(record:value('/count') > 0) ? '3' : '1'}"
} ]
}, {
"name" : "fieldAttributeConfigs",
"value" : [ {
"fieldToSet" : "/"
} ]
}, {
"name" : "stageOnRecordError",
"value" : "TO_ERROR"
}, {
"name" : "stageRequiredFields",
"value" : [ ]
}, {
"name" : "stageRecordPreconditions",
"value" : [ ]
} ],
"uiInfo" : {
"description" : "",
"label" : "Set Fields, Attributes",
"xPos" : 2041.2034912109375,
"yPos" : -83.14588356018066,
"stageType" : "PROCESSOR"
},
"inputLanes" : [ "HTTPClient_02OutputLane15269275492590" ],
"outputLanes" : [ "ExpressionEvaluator_01OutputLane15103392592010" ],
"eventLanes" : [ ],
"services" : [ ]
}, {
"instanceName" : "FieldTypeConverter_02",
"library" : "streamsets-datacollector-basic-lib",
"stageName" : "com_streamsets_pipeline_stage_processor_fieldtypeconverter_FieldTypeConverterDProcessor",
"stageVersion" : "2",
"configuration" : [ {
"name" : "convertBy",
"value" : "BY_FIELD"
}, {
"name" : "fieldTypeConverterConfigs",
"value" : [ {
"fields" : [ "/location/data/latitude", "/location/data/longitude" ],
"targetType" : "FLOAT",
"treatInputFieldAsDate" : false,
"dataLocale" : "en,US",
"scale" : -1,
"decimalScaleRoundingStrategy" : "ROUND_UNNECESSARY",
"dateFormat" : "YYYY_MM_DD",
"zonedDateTimeFormat" : "ISO_ZONED_DATE_TIME",
"encoding" : "UTF-8"
}, {
"fields" : [ "/lookup/data[0]/naicsCode" ],
"targetType" : "INTEGER",
"treatInputFieldAsDate" : false,
"dataLocale" : "en,US",
"scale" : -1,
"decimalScaleRoundingStrategy" : "ROUND_UNNECESSARY",
"dateFormat" : "YYYY_MM_DD",
"zonedDateTimeFormat" : "ISO_ZONED_DATE_TIME",
"encoding" : "UTF-8"
} ]
}, {
"name" : "wholeTypeConverterConfigs",
"value" : [ {
"sourceType" : "INTEGER",
"targetType" : "INTEGER",
"treatInputFieldAsDate" : false,
"dataLocale" : "en,US",
"scale" : -1,
"decimalScaleRoundingStrategy" : "ROUND_UNNECESSARY",
"dateFormat" : "YYYY_MM_DD",
"zonedDateTimeFormat" : "ISO_ZONED_DATE_TIME",
"encoding" : "UTF-8"
} ]
}, {
"name" : "stageOnRecordError",
"value" : "TO_ERROR"
}, {
"name" : "stageRequiredFields",
"value" : [ ]
}, {
"name" : "stageRecordPreconditions",
"value" : [ ]
} ],
"uiInfo" : {
"outputStreamLabels" : null,
"yPos" : -86.61127853393555,
"stageType" : "PROCESSOR",
"rawSource" : null,
"description" : "",
"label" : "Convert Field Types",
"xPos" : 2247.254638671875
},
"inputLanes" : [ "ExpressionEvaluator_01OutputLane15103392592010" ],
"outputLanes" : [ "FieldTypeConverter_02OutputLane15371495385510" ],
"eventLanes" : [ ],
"services" : [ ]
}, {
"instanceName" : "JDBCProducer_03",
"library" : "streamsets-datacollector-jdbc-lib",
"stageName" : "com_streamsets_pipeline_stage_destination_jdbc_JdbcDTarget",
"stageVersion" : "6",
"configuration" : [ {
"name" : "schema",
"value" : ""
}, {
"name" : "tableNameTemplate",
"value" : "client"
}, {
"name" : "columnNames",
"value" : [ {
"paramValue" : "?",
"dataType" : "USE_COLUMN_TYPE",
"columnName" : "ip",
"field" : "/ip"
}, {
"paramValue" : "?",
"dataType" : "USE_COLUMN_TYPE",
"columnName" : "city",
"field" : "/location/data/city"
}, {
"paramValue" : "?",
"dataType" : "USE_COLUMN_TYPE",
"columnName" : "region",
"field" : "/location/data/region"
}, {
"paramValue" : "?",
"dataType" : "USE_COLUMN_TYPE",
"columnName" : "countryShort",
"field" : "/location/data/countryShort"
}, {
"paramValue" : "?",
"columnName" : "latitude",
"dataType" : "USE_COLUMN_TYPE",
"field" : "/location/data/latitude"
}, {
"paramValue" : "?",
"dataType" : "USE_COLUMN_TYPE",
"columnName" : "longitude",
"field" : "/location/data/longitude"
}, {
"paramValue" : "?",
"dataType" : "USE_COLUMN_TYPE",
"columnName" : "org",
"field" : "/lookup/data[0]/name"
}, {
"paramValue" : "?",
"dataType" : "USE_COLUMN_TYPE",
"columnName" : "website",
"field" : "/lookup/data[0]/website"
}, {
"paramValue" : "?",
"dataType" : "USE_COLUMN_TYPE",
"columnName" : "regionShort",
"field" : "/location/data/regionShort"
}, {
"paramValue" : "?",
"columnName" : "country",
"field" : "/location/data/country",
"dataType" : "USE_COLUMN_TYPE"
}, {
"paramValue" : "?",
"dataType" : "USE_COLUMN_TYPE",
"columnName" : "employees",
"field" : "/lookup/data[0]/employees"
}, {
"paramValue" : "?",
"dataType" : "USE_COLUMN_TYPE",
"columnName" : "revenue",
"field" : "/lookup/data[0]/revenue"
}, {
"paramValue" : "?",
"dataType" : "USE_COLUMN_TYPE",
"columnName" : "sicGroup",
"field" : "/lookup/data[0]/sicGroup"
}, {
"paramValue" : "?",
"dataType" : "USE_COLUMN_TYPE",
"columnName" : "category",
"field" : "/lookup/data[0]/category"
}, {
"paramValue" : "?",
"dataType" : "USE_COLUMN_TYPE",
"columnName" : "sicCode",
"field" : "/lookup/data[0]/sicCode"
}, {
"paramValue" : "?",
"dataType" : "USE_COLUMN_TYPE",
"columnName" : "naicsGroup",
"field" : "/lookup/data[0]/naicsGroup"
}, {
"paramValue" : "?",
"dataType" : "USE_COLUMN_TYPE",
"columnName" : "category2",
"field" : "/lookup/data[0]/category2"
}, {
"paramValue" : "?",
"dataType" : "USE_COLUMN_TYPE",
"columnName" : "naicsCode",
"field" : "/lookup/data[0]/naicsCode"
}, {
"paramValue" : "?",
"dataType" : "USE_COLUMN_TYPE",
"columnName" : "stockSymbol",
"field" : "/lookup/data[0]/stockSymbol"
}, {
"paramValue" : "?",
"dataType" : "USE_COLUMN_TYPE",
"columnName" : "facebook",
"field" : "/lookup/data[0]/facebook"
}, {
"paramValue" : "?",
"dataType" : "USE_COLUMN_TYPE",
"columnName" : "twitter",
"field" : "/lookup/data[0]/twitter"
}, {
"paramValue" : "?",
"dataType" : "USE_COLUMN_TYPE",
"columnName" : "linkedIn",
"field" : "/lookup/data[0]/linkedIn"
}, {
"paramValue" : "?",
"dataType" : "USE_COLUMN_TYPE",
"columnName" : "confidence",
"field" : "/lookup/data[0]/confidence"
}, {
"paramValue" : "?",
"dataType" : "USE_COLUMN_TYPE",
"columnName" : "isISP",
"field" : "/lookup/data[0]/isISP"
}, {
"paramValue" : "?",
"dataType" : "USE_COLUMN_TYPE",
"columnName" : "isWifi",
"field" : "/lookup/data[0]/isWifi"
} ]
}, {
"name" : "encloseTableName",
"value" : false
}, {
"name" : "changeLogFormat",
"value" : "NONE"
}, {
"name" : "defaultOperation",
"value" : "INSERT"
}, {
"name" : "unsupportedAction",
"value" : "DISCARD"
}, {
"name" : "useMultiRowInsert",
"value" : false
}, {
"name" : "maxPrepStmtParameters",
"value" : -1
}, {
"name" : "maxPrepStmtCache",
"value" : -1
}, {
"name" : "rollbackOnError",
"value" : false
}, {
"name" : "hikariConfigBean.connectionString",
"value" : "${JDBC_URI}?useSSL=false"
}, {
"name" : "hikariConfigBean.useCredentials",
"value" : true
}, {
"name" : "hikariConfigBean.username",
"value" : "${JDBC_USER}"
}, {
"name" : "hikariConfigBean.password",
"value" : "${JDBC_PASSWORD}"
}, {
"name" : "hikariConfigBean.driverProperties",
"value" : [ ]
}, {
"name" : "hikariConfigBean.driverClassName",
"value" : null
}, {
"name" : "hikariConfigBean.connectionTestQuery",
"value" : null
}, {
"name" : "hikariConfigBean.maximumPoolSize",
"value" : 1
}, {
"name" : "hikariConfigBean.minIdle",
"value" : 1
}, {
"name" : "hikariConfigBean.connectionTimeout",
"value" : "${30 * SECONDS}"
}, {
"name" : "hikariConfigBean.idleTimeout",
"value" : "${10 * MINUTES}"
}, {
"name" : "hikariConfigBean.maxLifetime",
"value" : "${30 * MINUTES}"
}, {
"name" : "hikariConfigBean.transactionIsolation",
"value" : "DEFAULT"