Project Ouroboros Pipeline - see https://www.slideshare.net/metadaddy/project-ouroboros-using-streamsets-data-collector-to-help-manage-the-streamsets-open-source-community
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "pipelineConfig" : { | |
| "schemaVersion" : 6, | |
| "version" : 12, | |
| "pipelineId" : "Metrics635c20ed-872c-48d7-87fb-7043ab937ae6", | |
| "title" : "Metrics", | |
| "description" : "", | |
| "uuid" : "08bce59b-151b-47d4-97dc-5d4f58da9672", | |
| "configuration" : [ { | |
| "name" : "executionMode", | |
| "value" : "STANDALONE" | |
| }, { | |
| "name" : "deliveryGuarantee", | |
| "value" : "AT_LEAST_ONCE" | |
| }, { | |
| "name" : "shouldRetry", | |
| "value" : true | |
| }, { | |
| "name" : "retryAttempts", | |
| "value" : -1 | |
| }, { | |
| "name" : "notifyOnStates", | |
| "value" : [ "RUN_ERROR", "STOPPED", "FINISHED" ] | |
| }, { | |
| "name" : "emailIDs", | |
| "value" : [ ] | |
| }, { | |
| "name" : "constants", | |
| "value" : [ { | |
| "key" : "JDBC_URI", | |
| "value" : "jdbc:mysql://mysql.example.com/dbname" | |
| }, { | |
| "key" : "JDBC_USER", | |
| "value" : "" | |
| }, { | |
| "key" : "JDBC_PASSWORD", | |
| "value" : "" | |
| }, { | |
| "key" : "AWS_KEY", | |
| "value" : "" | |
| }, { | |
| "key" : "AWS_SECRET", | |
| "value" : "" | |
| }, { | |
| "key" : "KICKFIRE_KEY", | |
| "value" : "" | |
| } ] | |
| }, { | |
| "name" : "badRecordsHandling", | |
| "value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_recordstolocalfilesystem_ToErrorLocalFSDTarget::1" | |
| }, { | |
| "name" : "clusterSlaveMemory", | |
| "value" : 1024 | |
| }, { | |
| "name" : "clusterSlaveJavaOpts", | |
| "value" : "-XX:+UseConcMarkSweepGC -XX:+UseParNewGC -Dlog4j.debug" | |
| }, { | |
| "name" : "clusterLauncherEnv", | |
| "value" : [ ] | |
| }, { | |
| "name" : "mesosDispatcherURL", | |
| "value" : null | |
| }, { | |
| "name" : "hdfsS3ConfDir", | |
| "value" : null | |
| }, { | |
| "name" : "rateLimit", | |
| "value" : 0 | |
| }, { | |
| "name" : "maxRunners", | |
| "value" : 0 | |
| }, { | |
| "name" : "webhookConfigs", | |
| "value" : [ ] | |
| }, { | |
| "name" : "statsAggregatorStage", | |
| "value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_StatsDpmDirectlyDTarget::1" | |
| }, { | |
| "name" : "workerCount", | |
| "value" : 0 | |
| }, { | |
| "name" : "startEventStage", | |
| "value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1" | |
| }, { | |
| "name" : "stopEventStage", | |
| "value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1" | |
| }, { | |
| "name" : "errorRecordPolicy", | |
| "value" : "ORIGINAL_RECORD" | |
| }, { | |
| "name" : "shouldCreateFailureSnapshot", | |
| "value" : true | |
| }, { | |
| "name" : "sparkConfigs", | |
| "value" : [ ] | |
| }, { | |
| "name" : "edgeHttpUrl", | |
| "value" : "http://localhost:18633" | |
| }, { | |
| "name" : "runnerIdleTIme", | |
| "value" : 60 | |
| }, { | |
| "name" : "testOriginStage", | |
| "value" : "streamsets-datacollector-dev-lib::com_streamsets_pipeline_stage_devtest_rawdata_RawDataDSource::3" | |
| }, { | |
| "name" : "logLevel", | |
| "value" : "INFO" | |
| }, { | |
| "name" : "amazonEMRConfig.userRegion", | |
| "value" : null | |
| }, { | |
| "name" : "amazonEMRConfig.userRegionCustom", | |
| "value" : null | |
| }, { | |
| "name" : "amazonEMRConfig.accessKey", | |
| "value" : "" | |
| }, { | |
| "name" : "amazonEMRConfig.secretKey", | |
| "value" : "" | |
| }, { | |
| "name" : "amazonEMRConfig.s3StagingUri", | |
| "value" : null | |
| }, { | |
| "name" : "amazonEMRConfig.provisionNewCluster", | |
| "value" : false | |
| }, { | |
| "name" : "amazonEMRConfig.clusterId", | |
| "value" : null | |
| }, { | |
| "name" : "amazonEMRConfig.clusterPrefix", | |
| "value" : null | |
| }, { | |
| "name" : "amazonEMRConfig.terminateCluster", | |
| "value" : false | |
| }, { | |
| "name" : "amazonEMRConfig.loggingEnabled", | |
| "value" : true | |
| }, { | |
| "name" : "amazonEMRConfig.s3LogUri", | |
| "value" : null | |
| }, { | |
| "name" : "amazonEMRConfig.enableEMRDebugging", | |
| "value" : true | |
| }, { | |
| "name" : "amazonEMRConfig.serviceRole", | |
| "value" : "EMR_DefaultRole" | |
| }, { | |
| "name" : "amazonEMRConfig.jobFlowRole", | |
| "value" : "EMR_EC2_DefaultRole" | |
| }, { | |
| "name" : "amazonEMRConfig.visibleToAllUsers", | |
| "value" : true | |
| }, { | |
| "name" : "amazonEMRConfig.ec2SubnetId", | |
| "value" : null | |
| }, { | |
| "name" : "amazonEMRConfig.masterSecurityGroup", | |
| "value" : null | |
| }, { | |
| "name" : "amazonEMRConfig.slaveSecurityGroup", | |
| "value" : null | |
| }, { | |
| "name" : "amazonEMRConfig.instanceCount", | |
| "value" : 2 | |
| }, { | |
| "name" : "amazonEMRConfig.masterInstanceType", | |
| "value" : null | |
| }, { | |
| "name" : "amazonEMRConfig.masterInstanceTypeCustom", | |
| "value" : null | |
| }, { | |
| "name" : "amazonEMRConfig.slaveInstanceType", | |
| "value" : null | |
| }, { | |
| "name" : "amazonEMRConfig.slaveInstanceTypeCustom", | |
| "value" : null | |
| } ], | |
| "uiInfo" : { | |
| "previewConfig" : { | |
| "previewSource" : "CONFIGURED_SOURCE", | |
| "batchSize" : 10, | |
| "timeout" : "100000", | |
| "writeToDestinations" : false, | |
| "showHeader" : false, | |
| "showFieldType" : true, | |
| "rememberMe" : false | |
| } | |
| }, | |
| "fragments" : [ ], | |
| "stages" : [ { | |
| "instanceName" : "AmazonS3_01", | |
| "library" : "streamsets-datacollector-aws-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_origin_s3_AmazonS3DSource", | |
| "stageVersion" : "11", | |
| "configuration" : [ { | |
| "name" : "s3ConfigBean.basicConfig.maxBatchSize", | |
| "value" : 1000 | |
| }, { | |
| "name" : "s3ConfigBean.basicConfig.maxWaitTime", | |
| "value" : 2000 | |
| }, { | |
| "name" : "s3ConfigBean.sseConfig.useCustomerSSEKey", | |
| "value" : false | |
| }, { | |
| "name" : "s3ConfigBean.sseConfig.customerKey", | |
| "value" : "" | |
| }, { | |
| "name" : "s3ConfigBean.sseConfig.customerKeyMd5", | |
| "value" : "" | |
| }, { | |
| "name" : "s3ConfigBean.proxyConfig.useProxy", | |
| "value" : false | |
| }, { | |
| "name" : "s3ConfigBean.proxyConfig.proxyHost", | |
| "value" : null | |
| }, { | |
| "name" : "s3ConfigBean.proxyConfig.proxyPort", | |
| "value" : null | |
| }, { | |
| "name" : "s3ConfigBean.proxyConfig.proxyUser", | |
| "value" : "" | |
| }, { | |
| "name" : "s3ConfigBean.proxyConfig.proxyPassword", | |
| "value" : "" | |
| }, { | |
| "name" : "s3ConfigBean.errorConfig.errorHandlingOption", | |
| "value" : "NONE" | |
| }, { | |
| "name" : "s3ConfigBean.errorConfig.archivingOption", | |
| "value" : "MOVE_TO_PREFIX" | |
| }, { | |
| "name" : "s3ConfigBean.errorConfig.errorPrefix", | |
| "value" : null | |
| }, { | |
| "name" : "s3ConfigBean.errorConfig.errorBucket", | |
| "value" : null | |
| }, { | |
| "name" : "s3ConfigBean.postProcessingConfig.postProcessing", | |
| "value" : "NONE" | |
| }, { | |
| "name" : "s3ConfigBean.postProcessingConfig.archivingOption", | |
| "value" : "MOVE_TO_PREFIX" | |
| }, { | |
| "name" : "s3ConfigBean.postProcessingConfig.postProcessPrefix", | |
| "value" : null | |
| }, { | |
| "name" : "s3ConfigBean.postProcessingConfig.postProcessBucket", | |
| "value" : null | |
| }, { | |
| "name" : "s3ConfigBean.s3FileConfig.prefixPattern", | |
| "value" : "*.log.gz" | |
| }, { | |
| "name" : "s3ConfigBean.s3FileConfig.objectOrdering", | |
| "value" : "TIMESTAMP" | |
| }, { | |
| "name" : "s3ConfigBean.s3FileConfig.poolSize", | |
| "value" : 100 | |
| }, { | |
| "name" : "s3ConfigBean.s3FileConfig.overrunLimit", | |
| "value" : 128 | |
| }, { | |
| "name" : "s3ConfigBean.s3Config.bucket", | |
| "value" : "streamsets-downloadlogs" | |
| }, { | |
| "name" : "s3ConfigBean.s3Config.awsConfig.awsAccessKeyId", | |
| "value" : "${AWS_KEY}" | |
| }, { | |
| "name" : "s3ConfigBean.s3Config.awsConfig.awsSecretAccessKey", | |
| "value" : "${AWS_SECRET}" | |
| }, { | |
| "name" : "s3ConfigBean.s3Config.region", | |
| "value" : "US_EAST_1" | |
| }, { | |
| "name" : "s3ConfigBean.s3Config.endpoint", | |
| "value" : null | |
| }, { | |
| "name" : "s3ConfigBean.s3Config.commonPrefix", | |
| "value" : "fastly-cdn" | |
| }, { | |
| "name" : "s3ConfigBean.s3Config.delimiter", | |
| "value" : "/" | |
| }, { | |
| "name" : "s3ConfigBean.enableMetaData", | |
| "value" : false | |
| }, { | |
| "name" : "stageOnRecordError", | |
| "value" : "STOP_PIPELINE" | |
| }, { | |
| "name" : "s3ConfigBean.proxyConfig.connectionTimeout", | |
| "value" : 10 | |
| }, { | |
| "name" : "s3ConfigBean.proxyConfig.socketTimeout", | |
| "value" : 50 | |
| }, { | |
| "name" : "s3ConfigBean.proxyConfig.retryCount", | |
| "value" : 3 | |
| }, { | |
| "name" : "s3ConfigBean.numberOfThreads", | |
| "value" : 1 | |
| } ], | |
| "uiInfo" : { | |
| "description" : "", | |
| "label" : "Fastly CDN logs", | |
| "xPos" : 60, | |
| "yPos" : 48.408103942871094, | |
| "stageType" : "SOURCE" | |
| }, | |
| "inputLanes" : [ ], | |
| "outputLanes" : [ "AmazonS3_01OutputLane14974564326410" ], | |
| "eventLanes" : [ ], | |
| "services" : [ { | |
| "service" : "com.streamsets.pipeline.api.service.dataformats.DataFormatParserService", | |
| "serviceVersion" : 1, | |
| "configuration" : [ { | |
| "name" : "dataFormat", | |
| "value" : "LOG" | |
| }, { | |
| "name" : "dataFormatConfig.compression", | |
| "value" : "COMPRESSED_FILE" | |
| }, { | |
| "name" : "dataFormatConfig.filePatternInArchive", | |
| "value" : "*" | |
| }, { | |
| "name" : "dataFormatConfig.charset", | |
| "value" : "UTF-8" | |
| }, { | |
| "name" : "dataFormatConfig.removeCtrlChars", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.textMaxLineLen", | |
| "value" : 1024 | |
| }, { | |
| "name" : "dataFormatConfig.useCustomDelimiter", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.customDelimiter", | |
| "value" : "\\r\\n" | |
| }, { | |
| "name" : "dataFormatConfig.includeCustomDelimiterInTheText", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.jsonContent", | |
| "value" : "MULTIPLE_OBJECTS" | |
| }, { | |
| "name" : "dataFormatConfig.jsonMaxObjectLen", | |
| "value" : 4096 | |
| }, { | |
| "name" : "dataFormatConfig.csvFileFormat", | |
| "value" : "CSV" | |
| }, { | |
| "name" : "dataFormatConfig.csvHeader", | |
| "value" : "NO_HEADER" | |
| }, { | |
| "name" : "dataFormatConfig.csvMaxObjectLen", | |
| "value" : 1024 | |
| }, { | |
| "name" : "dataFormatConfig.csvCustomDelimiter", | |
| "value" : "|" | |
| }, { | |
| "name" : "dataFormatConfig.csvCustomEscape", | |
| "value" : "\\" | |
| }, { | |
| "name" : "dataFormatConfig.csvCustomQuote", | |
| "value" : "\"" | |
| }, { | |
| "name" : "dataFormatConfig.csvEnableComments", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.csvCommentMarker", | |
| "value" : "#" | |
| }, { | |
| "name" : "dataFormatConfig.csvIgnoreEmptyLines", | |
| "value" : true | |
| }, { | |
| "name" : "dataFormatConfig.csvRecordType", | |
| "value" : "LIST_MAP" | |
| }, { | |
| "name" : "dataFormatConfig.csvSkipStartLines", | |
| "value" : 0 | |
| }, { | |
| "name" : "dataFormatConfig.parseNull", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.nullConstant", | |
| "value" : "\\\\N" | |
| }, { | |
| "name" : "dataFormatConfig.xmlRecordElement", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.includeFieldXpathAttributes", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.xPathNamespaceContext", | |
| "value" : [ ] | |
| }, { | |
| "name" : "dataFormatConfig.outputFieldAttributes", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.xmlMaxObjectLen", | |
| "value" : 4096 | |
| }, { | |
| "name" : "dataFormatConfig.logMode", | |
| "value" : "GROK" | |
| }, { | |
| "name" : "dataFormatConfig.logMaxObjectLen", | |
| "value" : 1024 | |
| }, { | |
| "name" : "dataFormatConfig.retainOriginalLine", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.customLogFormat", | |
| "value" : "%h %l %u %t \"%r\" %>s %b" | |
| }, { | |
| "name" : "dataFormatConfig.regex", | |
| "value" : "^(\\S+) (\\S+) (\\S+) \\[([\\w:/]+\\s[+\\-]\\d{4})\\] \"(\\S+) (\\S+) (\\S+)\" (\\d{3}) (\\d+)" | |
| }, { | |
| "name" : "dataFormatConfig.fieldPathsToGroupName", | |
| "value" : [ { | |
| "fieldPath" : "/", | |
| "group" : 1 | |
| } ] | |
| }, { | |
| "name" : "dataFormatConfig.grokPatternDefinition", | |
| "value" : "DATESTAMP_FASTLY %{DAY}, %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} %{WORD:tz}\nSIZE_OR_NULL (?:%{NUMBER:size}|\\(null\\))" | |
| }, { | |
| "name" : "dataFormatConfig.grokPattern", | |
| "value" : "<%{NUMBER:priority}>%{TIMESTAMP_ISO8601:timestamp} %{HOSTNAME:cachenode} %{WORD:logname}\\[%{NUMBER:pid}\\]: %{IP:ip} \"-\" \"-\" %{DATESTAMP_FASTLY:datestamp} %{WORD:verb} %{NOTSPACE:file} %{NUMBER:code} %{SIZE_OR_NULL}" | |
| }, { | |
| "name" : "dataFormatConfig.onParseError", | |
| "value" : "ERROR" | |
| }, { | |
| "name" : "dataFormatConfig.maxStackTraceLines", | |
| "value" : 50 | |
| }, { | |
| "name" : "dataFormatConfig.enableLog4jCustomLogFormat", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.log4jCustomLogFormat", | |
| "value" : "%r [%t] %-5p %c %x - %m%n" | |
| }, { | |
| "name" : "dataFormatConfig.avroSchemaSource", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.avroSchema", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.schemaRegistryUrls", | |
| "value" : [ ] | |
| }, { | |
| "name" : "dataFormatConfig.schemaLookupMode", | |
| "value" : "SUBJECT" | |
| }, { | |
| "name" : "dataFormatConfig.subject", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.schemaId", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.protoDescriptorFile", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.messageType", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.isDelimited", | |
| "value" : true | |
| }, { | |
| "name" : "dataFormatConfig.binaryMaxObjectLen", | |
| "value" : 1024 | |
| }, { | |
| "name" : "dataFormatConfig.datagramMode", | |
| "value" : "SYSLOG" | |
| }, { | |
| "name" : "dataFormatConfig.typesDbPath", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.convertTime", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.excludeInterval", | |
| "value" : true | |
| }, { | |
| "name" : "dataFormatConfig.authFilePath", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.wholeFileMaxObjectLen", | |
| "value" : 8192 | |
| }, { | |
| "name" : "dataFormatConfig.rateLimit", | |
| "value" : "-1" | |
| }, { | |
| "name" : "dataFormatConfig.verifyChecksum", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.csvAllowExtraColumns", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.csvExtraColumnPrefix", | |
| "value" : "_extra_" | |
| }, { | |
| "name" : "dataFormatConfig.netflowOutputValuesMode", | |
| "value" : "RAW_AND_INTERPRETED" | |
| }, { | |
| "name" : "dataFormatConfig.maxTemplateCacheSize", | |
| "value" : -1 | |
| }, { | |
| "name" : "dataFormatConfig.templateCacheTimeoutMs", | |
| "value" : -1 | |
| }, { | |
| "name" : "dataFormatConfig.netflowOutputValuesModeDatagram", | |
| "value" : "RAW_AND_INTERPRETED" | |
| }, { | |
| "name" : "dataFormatConfig.maxTemplateCacheSizeDatagram", | |
| "value" : -1 | |
| }, { | |
| "name" : "dataFormatConfig.templateCacheTimeoutMsDatagram", | |
| "value" : -1 | |
| } ] | |
| } ] | |
| }, { | |
| "instanceName" : "FieldTypeConverter_01", | |
| "library" : "streamsets-datacollector-basic-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_processor_fieldtypeconverter_FieldTypeConverterDProcessor", | |
| "stageVersion" : "2", | |
| "configuration" : [ { | |
| "name" : "convertBy", | |
| "value" : "BY_FIELD" | |
| }, { | |
| "name" : "fieldTypeConverterConfigs", | |
| "value" : [ { | |
| "fields" : [ "/size", "/pid", "/code" ], | |
| "targetType" : "INTEGER", | |
| "treatInputFieldAsDate" : false, | |
| "dataLocale" : "en,US", | |
| "scale" : -1, | |
| "decimalScaleRoundingStrategy" : "ROUND_UNNECESSARY", | |
| "dateFormat" : "YYYY_MM_DD", | |
| "encoding" : "UTF-8", | |
| "zonedDateTimeFormat" : "ISO_ZONED_DATE_TIME", | |
| "otherZonedDateTimeFormat" : "" | |
| }, { | |
| "fields" : [ "/timestamp" ], | |
| "targetType" : "DATETIME", | |
| "treatInputFieldAsDate" : false, | |
| "dataLocale" : "en,US", | |
| "scale" : -1, | |
| "decimalScaleRoundingStrategy" : "ROUND_UNNECESSARY", | |
| "dateFormat" : "OTHER", | |
| "encoding" : "UTF-8", | |
| "otherDateFormat" : "yyyy-MM-dd'T'HH:mm:ss'Z'", | |
| "zonedDateTimeFormat" : "ISO_ZONED_DATE_TIME", | |
| "otherZonedDateTimeFormat" : "" | |
| } ] | |
| }, { | |
| "name" : "wholeTypeConverterConfigs", | |
| "value" : [ { | |
| "sourceType" : "INTEGER", | |
| "targetType" : "INTEGER", | |
| "treatInputFieldAsDate" : false, | |
| "dataLocale" : "en,US", | |
| "scale" : -1, | |
| "decimalScaleRoundingStrategy" : "ROUND_UNNECESSARY", | |
| "dateFormat" : "YYYY_MM_DD", | |
| "encoding" : "UTF-8", | |
| "zonedDateTimeFormat" : "ISO_ZONED_DATE_TIME", | |
| "otherZonedDateTimeFormat" : "" | |
| } ] | |
| }, { | |
| "name" : "stageOnRecordError", | |
| "value" : "TO_ERROR" | |
| }, { | |
| "name" : "stageRequiredFields", | |
| "value" : [ ] | |
| }, { | |
| "name" : "stageRecordPreconditions", | |
| "value" : [ ] | |
| } ], | |
| "uiInfo" : { | |
| "description" : "", | |
| "label" : "Convert date, integers", | |
| "xPos" : 280, | |
| "yPos" : 50, | |
| "stageType" : "PROCESSOR" | |
| }, | |
| "inputLanes" : [ "AmazonS3_01OutputLane14974564326410" ], | |
| "outputLanes" : [ "FieldTypeConverter_01OutputLane14974687038250" ], | |
| "eventLanes" : [ ], | |
| "services" : [ ] | |
| }, { | |
| "instanceName" : "StreamSelector_01", | |
| "library" : "streamsets-datacollector-basic-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_processor_selector_SelectorDProcessor", | |
| "stageVersion" : "1", | |
| "configuration" : [ { | |
| "name" : "lanePredicates", | |
| "value" : [ { | |
| "outputLane" : "StreamSelector_01OutputLane1499904663845", | |
| "predicate" : "${str:endsWith(record:value('/file'), '.parcel') || str:endsWith(record:value('/file'), '.zip') || str:endsWith(record:value('/file'), '.rpm') || str:endsWith(record:value('/file'), '.tgz')}" | |
| }, { | |
| "outputLane" : "StreamSelector_01OutputLane1499903680944", | |
| "predicate" : "${true}" | |
| }, { | |
| "outputLane" : "StreamSelector_01OutputLane1499903546028", | |
| "predicate" : "default" | |
| } ] | |
| }, { | |
| "name" : "stageOnRecordError", | |
| "value" : "TO_ERROR" | |
| }, { | |
| "name" : "stageRequiredFields", | |
| "value" : [ ] | |
| }, { | |
| "name" : "stageRecordPreconditions", | |
| "value" : [ ] | |
| } ], | |
| "uiInfo" : { | |
| "description" : "", | |
| "label" : "File download?", | |
| "xPos" : 500, | |
| "yPos" : 140, | |
| "stageType" : "PROCESSOR" | |
| }, | |
| "inputLanes" : [ "FieldTypeConverter_01OutputLane14974687038250" ], | |
| "outputLanes" : [ "StreamSelector_01OutputLane1499904663845", "StreamSelector_01OutputLane1499903680944", "StreamSelector_01OutputLane1499903546028" ], | |
| "eventLanes" : [ ], | |
| "services" : [ ] | |
| }, { | |
| "instanceName" : "JDBCProducer_02", | |
| "library" : "streamsets-datacollector-jdbc-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_destination_jdbc_JdbcDTarget", | |
| "stageVersion" : "6", | |
| "configuration" : [ { | |
| "name" : "schema", | |
| "value" : "" | |
| }, { | |
| "name" : "tableNameTemplate", | |
| "value" : "download" | |
| }, { | |
| "name" : "columnNames", | |
| "value" : [ { | |
| "paramValue" : "?", | |
| "dataType" : "USE_COLUMN_TYPE", | |
| "columnName" : "ip", | |
| "field" : "/ip" | |
| }, { | |
| "paramValue" : "?", | |
| "dataType" : "USE_COLUMN_TYPE", | |
| "columnName" : "date", | |
| "field" : "/timestamp" | |
| }, { | |
| "paramValue" : "?", | |
| "dataType" : "USE_COLUMN_TYPE", | |
| "columnName" : "file", | |
| "field" : "/file" | |
| } ] | |
| }, { | |
| "name" : "encloseTableName", | |
| "value" : false | |
| }, { | |
| "name" : "changeLogFormat", | |
| "value" : "NONE" | |
| }, { | |
| "name" : "defaultOperation", | |
| "value" : "INSERT" | |
| }, { | |
| "name" : "unsupportedAction", | |
| "value" : "DISCARD" | |
| }, { | |
| "name" : "useMultiRowInsert", | |
| "value" : false | |
| }, { | |
| "name" : "maxPrepStmtParameters", | |
| "value" : -1 | |
| }, { | |
| "name" : "maxPrepStmtCache", | |
| "value" : -1 | |
| }, { | |
| "name" : "rollbackOnError", | |
| "value" : false | |
| }, { | |
| "name" : "hikariConfigBean.connectionString", | |
| "value" : "${JDBC_URI}?useSSL=false" | |
| }, { | |
| "name" : "hikariConfigBean.useCredentials", | |
| "value" : true | |
| }, { | |
| "name" : "hikariConfigBean.username", | |
| "value" : "${JDBC_USER}" | |
| }, { | |
| "name" : "hikariConfigBean.password", | |
| "value" : "${JDBC_PASSWORD}" | |
| }, { | |
| "name" : "hikariConfigBean.driverProperties", | |
| "value" : [ ] | |
| }, { | |
| "name" : "hikariConfigBean.driverClassName", | |
| "value" : null | |
| }, { | |
| "name" : "hikariConfigBean.connectionTestQuery", | |
| "value" : null | |
| }, { | |
| "name" : "hikariConfigBean.maximumPoolSize", | |
| "value" : 1 | |
| }, { | |
| "name" : "hikariConfigBean.minIdle", | |
| "value" : 1 | |
| }, { | |
| "name" : "hikariConfigBean.connectionTimeout", | |
| "value" : "${30 * SECONDS}" | |
| }, { | |
| "name" : "hikariConfigBean.idleTimeout", | |
| "value" : "${10 * MINUTES}" | |
| }, { | |
| "name" : "hikariConfigBean.maxLifetime", | |
| "value" : "${30 * MINUTES}" | |
| }, { | |
| "name" : "hikariConfigBean.transactionIsolation", | |
| "value" : "TRANSACTION_READ_COMMITTED" | |
| }, { | |
| "name" : "stageOnRecordError", | |
| "value" : "TO_ERROR" | |
| }, { | |
| "name" : "stageRequiredFields", | |
| "value" : [ ] | |
| }, { | |
| "name" : "stageRecordPreconditions", | |
| "value" : [ ] | |
| }, { | |
| "name" : "hikariConfigBean.initialQuery", | |
| "value" : null | |
| } ], | |
| "uiInfo" : { | |
| "description" : "", | |
| "label" : "download table", | |
| "xPos" : 720, | |
| "yPos" : 40, | |
| "stageType" : "TARGET" | |
| }, | |
| "inputLanes" : [ "StreamSelector_01OutputLane1499904663845" ], | |
| "outputLanes" : [ ], | |
| "eventLanes" : [ ], | |
| "services" : [ ] | |
| }, { | |
| "instanceName" : "JDBCProducer_01", | |
| "library" : "streamsets-datacollector-jdbc-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_destination_jdbc_JdbcDTarget", | |
| "stageVersion" : "6", | |
| "configuration" : [ { | |
| "name" : "schema", | |
| "value" : "" | |
| }, { | |
| "name" : "tableNameTemplate", | |
| "value" : "log_entry" | |
| }, { | |
| "name" : "columnNames", | |
| "value" : [ { | |
| "paramValue" : "?", | |
| "dataType" : "USE_COLUMN_TYPE", | |
| "columnName" : "ip", | |
| "field" : "/ip" | |
| }, { | |
| "paramValue" : "?", | |
| "dataType" : "USE_COLUMN_TYPE", | |
| "columnName" : "date", | |
| "field" : "/timestamp" | |
| }, { | |
| "paramValue" : "?", | |
| "dataType" : "USE_COLUMN_TYPE", | |
| "columnName" : "verb", | |
| "field" : "/verb" | |
| }, { | |
| "paramValue" : "?", | |
| "dataType" : "USE_COLUMN_TYPE", | |
| "columnName" : "file", | |
| "field" : "/file" | |
| }, { | |
| "paramValue" : "?", | |
| "dataType" : "USE_COLUMN_TYPE", | |
| "columnName" : "code", | |
| "field" : "/code", | |
| "defaultValue" : "0" | |
| }, { | |
| "paramValue" : "?", | |
| "dataType" : "USE_COLUMN_TYPE", | |
| "columnName" : "size", | |
| "field" : "/size", | |
| "defaultValue" : "0" | |
| } ] | |
| }, { | |
| "name" : "encloseTableName", | |
| "value" : false | |
| }, { | |
| "name" : "changeLogFormat", | |
| "value" : "NONE" | |
| }, { | |
| "name" : "defaultOperation", | |
| "value" : "INSERT" | |
| }, { | |
| "name" : "unsupportedAction", | |
| "value" : "DISCARD" | |
| }, { | |
| "name" : "useMultiRowInsert", | |
| "value" : false | |
| }, { | |
| "name" : "maxPrepStmtParameters", | |
| "value" : -1 | |
| }, { | |
| "name" : "maxPrepStmtCache", | |
| "value" : -1 | |
| }, { | |
| "name" : "rollbackOnError", | |
| "value" : false | |
| }, { | |
| "name" : "hikariConfigBean.connectionString", | |
| "value" : "${JDBC_URI}?useSSL=false" | |
| }, { | |
| "name" : "hikariConfigBean.useCredentials", | |
| "value" : true | |
| }, { | |
| "name" : "hikariConfigBean.username", | |
| "value" : "${JDBC_USER}" | |
| }, { | |
| "name" : "hikariConfigBean.password", | |
| "value" : "${JDBC_PASSWORD}" | |
| }, { | |
| "name" : "hikariConfigBean.driverProperties", | |
| "value" : [ ] | |
| }, { | |
| "name" : "hikariConfigBean.driverClassName", | |
| "value" : null | |
| }, { | |
| "name" : "hikariConfigBean.connectionTestQuery", | |
| "value" : null | |
| }, { | |
| "name" : "hikariConfigBean.maximumPoolSize", | |
| "value" : 1 | |
| }, { | |
| "name" : "hikariConfigBean.minIdle", | |
| "value" : 1 | |
| }, { | |
| "name" : "hikariConfigBean.connectionTimeout", | |
| "value" : "${30 * SECONDS}" | |
| }, { | |
| "name" : "hikariConfigBean.idleTimeout", | |
| "value" : "${10 * MINUTES}" | |
| }, { | |
| "name" : "hikariConfigBean.maxLifetime", | |
| "value" : "${30 * MINUTES}" | |
| }, { | |
| "name" : "hikariConfigBean.transactionIsolation", | |
| "value" : "TRANSACTION_READ_COMMITTED" | |
| }, { | |
| "name" : "stageOnRecordError", | |
| "value" : "TO_ERROR" | |
| }, { | |
| "name" : "stageRequiredFields", | |
| "value" : [ ] | |
| }, { | |
| "name" : "stageRecordPreconditions", | |
| "value" : [ ] | |
| }, { | |
| "name" : "hikariConfigBean.initialQuery", | |
| "value" : null | |
| } ], | |
| "uiInfo" : { | |
| "description" : "", | |
| "label" : "log_entry table", | |
| "xPos" : 720, | |
| "yPos" : 170.0109405517578, | |
| "stageType" : "TARGET" | |
| }, | |
| "inputLanes" : [ "StreamSelector_01OutputLane1499903680944" ], | |
| "outputLanes" : [ ], | |
| "eventLanes" : [ ], | |
| "services" : [ ] | |
| }, { | |
| "instanceName" : "Trash_01", | |
| "library" : "streamsets-datacollector-basic-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_destination_devnull_NullDTarget", | |
| "stageVersion" : "1", | |
| "configuration" : [ ], | |
| "uiInfo" : { | |
| "description" : "", | |
| "label" : "Trash 1", | |
| "xPos" : 720, | |
| "yPos" : 300, | |
| "stageType" : "TARGET" | |
| }, | |
| "inputLanes" : [ "StreamSelector_01OutputLane1499903546028" ], | |
| "outputLanes" : [ ], | |
| "eventLanes" : [ ], | |
| "services" : [ ] | |
| }, { | |
| "instanceName" : "RecordDeduplicator_01", | |
| "library" : "streamsets-datacollector-basic-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_processor_dedup_DeDupDProcessor", | |
| "stageVersion" : "1", | |
| "configuration" : [ { | |
| "name" : "recordCountWindow", | |
| "value" : 1000000 | |
| }, { | |
| "name" : "timeWindowSecs", | |
| "value" : 0 | |
| }, { | |
| "name" : "compareFields", | |
| "value" : "SPECIFIED_FIELDS" | |
| }, { | |
| "name" : "fieldsToCompare", | |
| "value" : [ "/ip" ] | |
| }, { | |
| "name" : "stageRequiredFields", | |
| "value" : [ ] | |
| }, { | |
| "name" : "stageRecordPreconditions", | |
| "value" : [ ] | |
| } ], | |
| "uiInfo" : { | |
| "outputStreamLabels" : [ "Unique Records", "Duplicate Records" ], | |
| "yPos" : -94.90224838256836, | |
| "stageType" : "PROCESSOR", | |
| "description" : "", | |
| "label" : "Deduplicate IPs", | |
| "xPos" : 721.3631591796875 | |
| }, | |
| "inputLanes" : [ "StreamSelector_01OutputLane1499904663845" ], | |
| "outputLanes" : [ "RecordDeduplicator_01OutputLane15015103760830", "RecordDeduplicator_01OutputLane15015103760831" ], | |
| "eventLanes" : [ ], | |
| "services" : [ ] | |
| }, { | |
| "instanceName" : "Trash_04", | |
| "library" : "streamsets-datacollector-basic-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_destination_devnull_NullDTarget", | |
| "stageVersion" : "1", | |
| "configuration" : [ ], | |
| "uiInfo" : { | |
| "description" : "", | |
| "label" : "Trash 4", | |
| "xPos" : 935.0000610351562, | |
| "yPos" : 45, | |
| "stageType" : "TARGET" | |
| }, | |
| "inputLanes" : [ "RecordDeduplicator_01OutputLane15015103760831" ], | |
| "outputLanes" : [ ], | |
| "eventLanes" : [ ], | |
| "services" : [ ] | |
| }, { | |
| "instanceName" : "JDBCLookup_01", | |
| "library" : "streamsets-datacollector-jdbc-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_processor_jdbclookup_JdbcLookupDProcessor", | |
| "stageVersion" : "3", | |
| "configuration" : [ { | |
| "name" : "query", | |
| "value" : "SELECT COUNT(*) AS count, COUNT(website) AS website\nFROM client \nWHERE ip = '${record:value('/ip')}'" | |
| }, { | |
| "name" : "columnMappings", | |
| "value" : [ { | |
| "dataType" : "USE_COLUMN_TYPE", | |
| "columnName" : "count", | |
| "field" : "/count", | |
| "defaultValue" : "" | |
| } ] | |
| }, { | |
| "name" : "multipleValuesBehavior", | |
| "value" : "FIRST_ONLY" | |
| }, { | |
| "name" : "maxClobSize", | |
| "value" : 1000 | |
| }, { | |
| "name" : "maxBlobSize", | |
| "value" : 1000 | |
| }, { | |
| "name" : "hikariConfigBean.connectionString", | |
| "value" : "${JDBC_URI}?useSSL=false" | |
| }, { | |
| "name" : "hikariConfigBean.useCredentials", | |
| "value" : true | |
| }, { | |
| "name" : "hikariConfigBean.username", | |
| "value" : "${JDBC_USER}" | |
| }, { | |
| "name" : "hikariConfigBean.password", | |
| "value" : "${JDBC_PASSWORD}" | |
| }, { | |
| "name" : "hikariConfigBean.driverProperties", | |
| "value" : [ ] | |
| }, { | |
| "name" : "hikariConfigBean.driverClassName", | |
| "value" : null | |
| }, { | |
| "name" : "hikariConfigBean.connectionTestQuery", | |
| "value" : null | |
| }, { | |
| "name" : "hikariConfigBean.maximumPoolSize", | |
| "value" : 1 | |
| }, { | |
| "name" : "hikariConfigBean.minIdle", | |
| "value" : 1 | |
| }, { | |
| "name" : "hikariConfigBean.connectionTimeout", | |
| "value" : "${30 * SECONDS}" | |
| }, { | |
| "name" : "hikariConfigBean.idleTimeout", | |
| "value" : "${10 * MINUTES}" | |
| }, { | |
| "name" : "hikariConfigBean.maxLifetime", | |
| "value" : "${30 * MINUTES}" | |
| }, { | |
| "name" : "hikariConfigBean.autoCommit", | |
| "value" : false | |
| }, { | |
| "name" : "hikariConfigBean.readOnly", | |
| "value" : true | |
| }, { | |
| "name" : "hikariConfigBean.transactionIsolation", | |
| "value" : "DEFAULT" | |
| }, { | |
| "name" : "cacheConfig.enabled", | |
| "value" : false | |
| }, { | |
| "name" : "cacheConfig.maxSize", | |
| "value" : -1 | |
| }, { | |
| "name" : "cacheConfig.evictionPolicyType", | |
| "value" : null | |
| }, { | |
| "name" : "cacheConfig.expirationTime", | |
| "value" : 1 | |
| }, { | |
| "name" : "cacheConfig.timeUnit", | |
| "value" : "SECONDS" | |
| }, { | |
| "name" : "stageOnRecordError", | |
| "value" : "TO_ERROR" | |
| }, { | |
| "name" : "stageRequiredFields", | |
| "value" : [ ] | |
| }, { | |
| "name" : "stageRecordPreconditions", | |
| "value" : [ ] | |
| }, { | |
| "name" : "hikariConfigBean.initialQuery", | |
| "value" : null | |
| }, { | |
| "name" : "missingValuesBehavior", | |
| "value" : "SEND_TO_ERROR" | |
| }, { | |
| "name" : "cacheConfig.retryOnCacheMiss", | |
| "value" : false | |
| } ], | |
| "uiInfo" : { | |
| "description" : "", | |
| "label" : "client lookup", | |
| "xPos" : 936.6666259765625, | |
| "yPos" : -93.3333330154419, | |
| "stageType" : "PROCESSOR" | |
| }, | |
| "inputLanes" : [ "RecordDeduplicator_01OutputLane15015103760830" ], | |
| "outputLanes" : [ "JDBCLookup_01OutputLane15000865896940" ], | |
| "eventLanes" : [ ], | |
| "services" : [ ] | |
| }, { | |
| "instanceName" : "StreamSelector_02", | |
| "library" : "streamsets-datacollector-basic-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_processor_selector_SelectorDProcessor", | |
| "stageVersion" : "1", | |
| "configuration" : [ { | |
| "name" : "lanePredicates", | |
| "value" : [ { | |
| "outputLane" : "StreamSelector_02OutputLane15000866189680", | |
| "predicate" : "${record:value('/website') > 0}" | |
| }, { | |
| "outputLane" : "StreamSelector_02OutputLane15000866189681", | |
| "predicate" : "default" | |
| } ] | |
| }, { | |
| "name" : "stageOnRecordError", | |
| "value" : "TO_ERROR" | |
| }, { | |
| "name" : "stageRequiredFields", | |
| "value" : [ ] | |
| }, { | |
| "name" : "stageRecordPreconditions", | |
| "value" : [ ] | |
| } ], | |
| "uiInfo" : { | |
| "description" : "", | |
| "label" : "website is set?", | |
| "xPos" : 1143.86767578125, | |
| "yPos" : -94.99998092651367, | |
| "stageType" : "PROCESSOR" | |
| }, | |
| "inputLanes" : [ "JDBCLookup_01OutputLane15000865896940" ], | |
| "outputLanes" : [ "StreamSelector_02OutputLane15000866189680", "StreamSelector_02OutputLane15000866189681" ], | |
| "eventLanes" : [ ], | |
| "services" : [ ] | |
| }, { | |
| "instanceName" : "HTTPClient_01", | |
| "library" : "streamsets-datacollector-basic-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_processor_http_HttpDProcessor", | |
| "stageVersion" : "11", | |
| "configuration" : [ { | |
| "name" : "conf.dataFormatConfig.filePatternInArchive", | |
| "value" : "*" | |
| }, { | |
| "name" : "conf.dataFormatConfig.charset", | |
| "value" : "UTF-8" | |
| }, { | |
| "name" : "conf.dataFormatConfig.removeCtrlChars", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.textMaxLineLen", | |
| "value" : 1024 | |
| }, { | |
| "name" : "conf.dataFormatConfig.useCustomDelimiter", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.customDelimiter", | |
| "value" : "\\r\\n" | |
| }, { | |
| "name" : "conf.dataFormatConfig.includeCustomDelimiterInTheText", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.jsonMaxObjectLen", | |
| "value" : 4096 | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvFileFormat", | |
| "value" : "CSV" | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvHeader", | |
| "value" : "NO_HEADER" | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvMaxObjectLen", | |
| "value" : 1024 | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvCustomDelimiter", | |
| "value" : "|" | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvCustomEscape", | |
| "value" : "\\" | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvCustomQuote", | |
| "value" : "\"" | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvEnableComments", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvCommentMarker", | |
| "value" : "#" | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvIgnoreEmptyLines", | |
| "value" : true | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvRecordType", | |
| "value" : "LIST_MAP" | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvSkipStartLines", | |
| "value" : 0 | |
| }, { | |
| "name" : "conf.dataFormatConfig.parseNull", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.nullConstant", | |
| "value" : "\\\\N" | |
| }, { | |
| "name" : "conf.dataFormatConfig.xmlRecordElement", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.includeFieldXpathAttributes", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.xPathNamespaceContext", | |
| "value" : [ ] | |
| }, { | |
| "name" : "conf.dataFormatConfig.outputFieldAttributes", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.xmlMaxObjectLen", | |
| "value" : 4096 | |
| }, { | |
| "name" : "conf.dataFormatConfig.logMode", | |
| "value" : "COMMON_LOG_FORMAT" | |
| }, { | |
| "name" : "conf.dataFormatConfig.logMaxObjectLen", | |
| "value" : 1024 | |
| }, { | |
| "name" : "conf.dataFormatConfig.retainOriginalLine", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.customLogFormat", | |
| "value" : "%h %l %u %t \"%r\" %>s %b" | |
| }, { | |
| "name" : "conf.dataFormatConfig.regex", | |
| "value" : "^(\\S+) (\\S+) (\\S+) \\[([\\w:/]+\\s[+\\-]\\d{4})\\] \"(\\S+) (\\S+) (\\S+)\" (\\d{3}) (\\d+)" | |
| }, { | |
| "name" : "conf.dataFormatConfig.fieldPathsToGroupName", | |
| "value" : [ { | |
| "fieldPath" : "/", | |
| "group" : 1 | |
| } ] | |
| }, { | |
| "name" : "conf.dataFormatConfig.grokPatternDefinition", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.grokPattern", | |
| "value" : "%{COMMONAPACHELOG}" | |
| }, { | |
| "name" : "conf.dataFormatConfig.onParseError", | |
| "value" : "ERROR" | |
| }, { | |
| "name" : "conf.dataFormatConfig.maxStackTraceLines", | |
| "value" : 50 | |
| }, { | |
| "name" : "conf.dataFormatConfig.enableLog4jCustomLogFormat", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.log4jCustomLogFormat", | |
| "value" : "%r [%t] %-5p %c %x - %m%n" | |
| }, { | |
| "name" : "conf.dataFormatConfig.avroSchemaSource", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.avroSchema", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.schemaRegistryUrls", | |
| "value" : [ ] | |
| }, { | |
| "name" : "conf.dataFormatConfig.schemaLookupMode", | |
| "value" : "SUBJECT" | |
| }, { | |
| "name" : "conf.dataFormatConfig.subject", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.schemaId", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.protoDescriptorFile", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.messageType", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.isDelimited", | |
| "value" : true | |
| }, { | |
| "name" : "conf.dataFormatConfig.binaryMaxObjectLen", | |
| "value" : 1024 | |
| }, { | |
| "name" : "conf.dataFormatConfig.datagramMode", | |
| "value" : "SYSLOG" | |
| }, { | |
| "name" : "conf.dataFormatConfig.typesDbPath", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.convertTime", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.excludeInterval", | |
| "value" : true | |
| }, { | |
| "name" : "conf.dataFormatConfig.authFilePath", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.wholeFileMaxObjectLen", | |
| "value" : 8192 | |
| }, { | |
| "name" : "conf.dataFormatConfig.rateLimit", | |
| "value" : "-1" | |
| }, { | |
| "name" : "conf.dataFormatConfig.verifyChecksum", | |
| "value" : false | |
| }, { | |
| "name" : "conf.outputField", | |
| "value" : "/lookup" | |
| }, { | |
| "name" : "conf.headerOutputLocation", | |
| "value" : "HEADER" | |
| }, { | |
| "name" : "conf.headerAttributePrefix", | |
| "value" : null | |
| }, { | |
| "name" : "conf.headerOutputField", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormat", | |
| "value" : "JSON" | |
| }, { | |
| "name" : "conf.resourceUrl", | |
| "value" : "https://api.kickfire.com/v2/company:(all)?ip=${record:value('/ip')}&key=${KICKFIRE_KEY}" | |
| }, { | |
| "name" : "conf.headers", | |
| "value" : [ ] | |
| }, { | |
| "name" : "conf.httpMethod", | |
| "value" : "GET" | |
| }, { | |
| "name" : "conf.methodExpression", | |
| "value" : null | |
| }, { | |
| "name" : "conf.requestBody", | |
| "value" : null | |
| }, { | |
| "name" : "conf.defaultRequestContentType", | |
| "value" : "application/json" | |
| }, { | |
| "name" : "conf.client.transferEncoding", | |
| "value" : "CHUNKED" | |
| }, { | |
| "name" : "conf.client.httpCompression", | |
| "value" : "NONE" | |
| }, { | |
| "name" : "conf.client.connectTimeoutMillis", | |
| "value" : 0 | |
| }, { | |
| "name" : "conf.client.readTimeoutMillis", | |
| "value" : 0 | |
| }, { | |
| "name" : "conf.client.numThreads", | |
| "value" : 1 | |
| }, { | |
| "name" : "conf.client.authType", | |
| "value" : "NONE" | |
| }, { | |
| "name" : "conf.client.useOAuth2", | |
| "value" : false | |
| }, { | |
| "name" : "conf.client.oauth.consumerKey", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.client.oauth.consumerSecret", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.client.oauth.token", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.client.oauth.tokenSecret", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.client.oauth2.credentialsGrantType", | |
| "value" : null | |
| }, { | |
| "name" : "conf.client.oauth2.tokenUrl", | |
| "value" : null | |
| }, { | |
| "name" : "conf.client.oauth2.clientId", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.client.oauth2.clientSecret", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.client.oauth2.username", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.client.oauth2.password", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.client.oauth2.resourceOwnerClientId", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.client.oauth2.resourceOwnerClientSecret", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.client.oauth2.algorithm", | |
| "value" : "NONE" | |
| }, { | |
| "name" : "conf.client.oauth2.key", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.client.oauth2.jwtClaims", | |
| "value" : null | |
| }, { | |
| "name" : "conf.client.oauth2.transferEncoding", | |
| "value" : "BUFFERED" | |
| }, { | |
| "name" : "conf.client.oauth2.additionalValues", | |
| "value" : [ ] | |
| }, { | |
| "name" : "conf.client.basicAuth.username", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.client.basicAuth.password", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.client.useProxy", | |
| "value" : false | |
| }, { | |
| "name" : "conf.client.proxy.uri", | |
| "value" : null | |
| }, { | |
| "name" : "conf.client.proxy.username", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.client.proxy.password", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.client.tlsConfig.tlsEnabled", | |
| "value" : false | |
| }, { | |
| "name" : "conf.client.tlsConfig.trustStoreFilePath", | |
| "value" : null | |
| }, { | |
| "name" : "conf.client.tlsConfig.trustStoreType", | |
| "value" : "JKS" | |
| }, { | |
| "name" : "conf.client.tlsConfig.trustStorePassword", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.client.tlsConfig.trustStoreAlgorithm", | |
| "value" : "SunX509" | |
| }, { | |
| "name" : "conf.client.tlsConfig.useDefaultProtocols", | |
| "value" : true | |
| }, { | |
| "name" : "conf.client.tlsConfig.protocols", | |
| "value" : [ ] | |
| }, { | |
| "name" : "conf.client.tlsConfig.useDefaultCiperSuites", | |
| "value" : true | |
| }, { | |
| "name" : "conf.client.tlsConfig.cipherSuites", | |
| "value" : [ ] | |
| }, { | |
| "name" : "conf.maxRequestCompletionSecs", | |
| "value" : 60 | |
| }, { | |
| "name" : "stageOnRecordError", | |
| "value" : "STOP_PIPELINE" | |
| }, { | |
| "name" : "stageRequiredFields", | |
| "value" : [ ] | |
| }, { | |
| "name" : "stageRecordPreconditions", | |
| "value" : [ ] | |
| }, { | |
| "name" : "conf.rateLimit", | |
| "value" : 3000 | |
| }, { | |
| "name" : "conf.client.requestLoggingConfig.enableRequestLogging", | |
| "value" : false | |
| }, { | |
| "name" : "conf.client.requestLoggingConfig.logLevel", | |
| "value" : "FINE" | |
| }, { | |
| "name" : "conf.client.requestLoggingConfig.verbosity", | |
| "value" : "HEADERS_ONLY" | |
| }, { | |
| "name" : "conf.client.requestLoggingConfig.maxEntitySize", | |
| "value" : 0 | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvAllowExtraColumns", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvExtraColumnPrefix", | |
| "value" : "_extra_" | |
| }, { | |
| "name" : "conf.dataFormatConfig.netflowOutputValuesMode", | |
| "value" : "RAW_AND_INTERPRETED" | |
| }, { | |
| "name" : "conf.dataFormatConfig.maxTemplateCacheSize", | |
| "value" : -1 | |
| }, { | |
| "name" : "conf.dataFormatConfig.templateCacheTimeoutMs", | |
| "value" : -1 | |
| }, { | |
| "name" : "conf.dataFormatConfig.netflowOutputValuesModeDatagram", | |
| "value" : "RAW_AND_INTERPRETED" | |
| }, { | |
| "name" : "conf.dataFormatConfig.maxTemplateCacheSizeDatagram", | |
| "value" : -1 | |
| }, { | |
| "name" : "conf.dataFormatConfig.templateCacheTimeoutMsDatagram", | |
| "value" : -1 | |
| }, { | |
| "name" : "conf.client.tlsConfig.keyStoreFilePath", | |
| "value" : null | |
| }, { | |
| "name" : "conf.client.tlsConfig.keyStoreType", | |
| "value" : "JKS" | |
| }, { | |
| "name" : "conf.client.tlsConfig.keyStorePassword", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.client.tlsConfig.keyStoreAlgorithm", | |
| "value" : "SunX509" | |
| }, { | |
| "name" : "conf.dataFormatConfig.compression", | |
| "value" : "NONE" | |
| }, { | |
| "name" : "conf.dataFormatConfig.excelHeader", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.multiCharacterFieldDelimiter", | |
| "value" : "||" | |
| }, { | |
| "name" : "conf.dataFormatConfig.multiCharacterLineDelimiter", | |
| "value" : "${str:unescapeJava('\\\\n')}" | |
| } ], | |
| "uiInfo" : { | |
| "description" : "", | |
| "label" : "Kickfire Company Lookup", | |
| "xPos" : 1364.79052734375, | |
| "yPos" : 50.00000762939453, | |
| "stageType" : "PROCESSOR" | |
| }, | |
| "inputLanes" : [ "StreamSelector_02OutputLane15000866189681" ], | |
| "outputLanes" : [ "HTTPClient_01OutputLane15000866409770" ], | |
| "eventLanes" : [ ], | |
| "services" : [ ] | |
| }, { | |
| "instanceName" : "Trash_02", | |
| "library" : "streamsets-datacollector-basic-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_destination_devnull_NullDTarget", | |
| "stageVersion" : "1", | |
| "configuration" : [ ], | |
| "uiInfo" : { | |
| "description" : "", | |
| "label" : "Trash 2", | |
| "xPos" : 1363.3333740234375, | |
| "yPos" : -91.66665267944336, | |
| "stageType" : "TARGET" | |
| }, | |
| "inputLanes" : [ "StreamSelector_02OutputLane15000866189680" ], | |
| "outputLanes" : [ ], | |
| "eventLanes" : [ ], | |
| "services" : [ ] | |
| }, { | |
| "instanceName" : "StreamSelector_04", | |
| "library" : "streamsets-datacollector-basic-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_processor_selector_SelectorDProcessor", | |
| "stageVersion" : "1", | |
| "configuration" : [ { | |
| "name" : "lanePredicates", | |
| "value" : [ { | |
| "outputLane" : "StreamSelector_04OutputLane1500087032682", | |
| "predicate" : "${record:value('/lookup/status') == 'success'}" | |
| }, { | |
| "outputLane" : "StreamSelector_04OutputLane1500087006721", | |
| "predicate" : "default" | |
| } ] | |
| }, { | |
| "name" : "stageOnRecordError", | |
| "value" : "TO_ERROR" | |
| }, { | |
| "name" : "stageRequiredFields", | |
| "value" : [ ] | |
| }, { | |
| "name" : "stageRecordPreconditions", | |
| "value" : [ ] | |
| } ], | |
| "uiInfo" : { | |
| "description" : "", | |
| "label" : "Kickfire success?", | |
| "xPos" : 1565, | |
| "yPos" : 51.66667938232422, | |
| "stageType" : "PROCESSOR" | |
| }, | |
| "inputLanes" : [ "HTTPClient_01OutputLane15000866409770" ], | |
| "outputLanes" : [ "StreamSelector_04OutputLane1500087032682", "StreamSelector_04OutputLane1500087006721" ], | |
| "eventLanes" : [ ], | |
| "services" : [ ] | |
| }, { | |
| "instanceName" : "ToError_01", | |
| "library" : "streamsets-datacollector-basic-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_destination_toerror_ToErrorDTarget", | |
| "stageVersion" : "1", | |
| "configuration" : [ ], | |
| "uiInfo" : { | |
| "description" : "", | |
| "label" : "To Error 1", | |
| "xPos" : 1818.333251953125, | |
| "yPos" : 178.33334350585938, | |
| "stageType" : "TARGET" | |
| }, | |
| "inputLanes" : [ "StreamSelector_04OutputLane1500087006721" ], | |
| "outputLanes" : [ ], | |
| "eventLanes" : [ ], | |
| "services" : [ ] | |
| }, { | |
| "instanceName" : "RecordDeduplicator_02", | |
| "library" : "streamsets-datacollector-basic-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_processor_dedup_DeDupDProcessor", | |
| "stageVersion" : "1", | |
| "configuration" : [ { | |
| "name" : "recordCountWindow", | |
| "value" : 1000000 | |
| }, { | |
| "name" : "timeWindowSecs", | |
| "value" : 0 | |
| }, { | |
| "name" : "compareFields", | |
| "value" : "SPECIFIED_FIELDS" | |
| }, { | |
| "name" : "fieldsToCompare", | |
| "value" : [ "/lookup/data[0]/name" ] | |
| }, { | |
| "name" : "stageRequiredFields", | |
| "value" : [ "/lookup/data[0]/name" ] | |
| }, { | |
| "name" : "stageRecordPreconditions", | |
| "value" : [ ] | |
| } ], | |
| "uiInfo" : { | |
| "outputStreamLabels" : [ "Unique Records", "Duplicate Records" ], | |
| "yPos" : 46.66667175292969, | |
| "stageType" : "PROCESSOR", | |
| "description" : "", | |
| "label" : "Deduplicate Company Names", | |
| "xPos" : 1811.666748046875 | |
| }, | |
| "inputLanes" : [ "StreamSelector_04OutputLane1500087032682" ], | |
| "outputLanes" : [ "RecordDeduplicator_02OutputLane15015108315570", "RecordDeduplicator_02OutputLane15015108315571" ], | |
| "eventLanes" : [ ], | |
| "services" : [ ] | |
| }, { | |
| "instanceName" : "JDBCLookup_02", | |
| "library" : "streamsets-datacollector-jdbc-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_processor_jdbclookup_JdbcLookupDProcessor", | |
| "stageVersion" : "3", | |
| "configuration" : [ { | |
| "name" : "query", | |
| "value" : "SELECT COUNT(*) as count FROM org_company WHERE org = '${record:value('/lookup/data[0]/name')}'" | |
| }, { | |
| "name" : "columnMappings", | |
| "value" : [ { | |
| "dataType" : "USE_COLUMN_TYPE", | |
| "columnName" : "count", | |
| "field" : "/count", | |
| "defaultValue" : "" | |
| } ] | |
| }, { | |
| "name" : "multipleValuesBehavior", | |
| "value" : "FIRST_ONLY" | |
| }, { | |
| "name" : "maxClobSize", | |
| "value" : 1000 | |
| }, { | |
| "name" : "maxBlobSize", | |
| "value" : 1000 | |
| }, { | |
| "name" : "hikariConfigBean.connectionString", | |
| "value" : "${JDBC_URI}?useSSL=false" | |
| }, { | |
| "name" : "hikariConfigBean.useCredentials", | |
| "value" : true | |
| }, { | |
| "name" : "hikariConfigBean.username", | |
| "value" : "${JDBC_USER}" | |
| }, { | |
| "name" : "hikariConfigBean.password", | |
| "value" : "${JDBC_PASSWORD}" | |
| }, { | |
| "name" : "hikariConfigBean.driverProperties", | |
| "value" : [ ] | |
| }, { | |
| "name" : "hikariConfigBean.driverClassName", | |
| "value" : null | |
| }, { | |
| "name" : "hikariConfigBean.connectionTestQuery", | |
| "value" : null | |
| }, { | |
| "name" : "hikariConfigBean.maximumPoolSize", | |
| "value" : 1 | |
| }, { | |
| "name" : "hikariConfigBean.minIdle", | |
| "value" : 1 | |
| }, { | |
| "name" : "hikariConfigBean.connectionTimeout", | |
| "value" : "${30 * SECONDS}" | |
| }, { | |
| "name" : "hikariConfigBean.idleTimeout", | |
| "value" : "${10 * MINUTES}" | |
| }, { | |
| "name" : "hikariConfigBean.maxLifetime", | |
| "value" : "${30 * MINUTES}" | |
| }, { | |
| "name" : "hikariConfigBean.autoCommit", | |
| "value" : false | |
| }, { | |
| "name" : "hikariConfigBean.readOnly", | |
| "value" : true | |
| }, { | |
| "name" : "hikariConfigBean.transactionIsolation", | |
| "value" : "DEFAULT" | |
| }, { | |
| "name" : "cacheConfig.enabled", | |
| "value" : false | |
| }, { | |
| "name" : "cacheConfig.maxSize", | |
| "value" : -1 | |
| }, { | |
| "name" : "cacheConfig.evictionPolicyType", | |
| "value" : null | |
| }, { | |
| "name" : "cacheConfig.expirationTime", | |
| "value" : 1 | |
| }, { | |
| "name" : "cacheConfig.timeUnit", | |
| "value" : "SECONDS" | |
| }, { | |
| "name" : "stageOnRecordError", | |
| "value" : "TO_ERROR" | |
| }, { | |
| "name" : "stageRequiredFields", | |
| "value" : [ ] | |
| }, { | |
| "name" : "stageRecordPreconditions", | |
| "value" : [ ] | |
| }, { | |
| "name" : "hikariConfigBean.initialQuery", | |
| "value" : null | |
| }, { | |
| "name" : "missingValuesBehavior", | |
| "value" : "SEND_TO_ERROR" | |
| }, { | |
| "name" : "cacheConfig.retryOnCacheMiss", | |
| "value" : false | |
| } ], | |
| "uiInfo" : { | |
| "description" : "", | |
| "label" : "org/company lookup", | |
| "xPos" : 2041.666748046875, | |
| "yPos" : 50.00001525878906, | |
| "stageType" : "PROCESSOR" | |
| }, | |
| "inputLanes" : [ "RecordDeduplicator_02OutputLane15015108315570" ], | |
| "outputLanes" : [ "JDBCLookup_02OutputLane15000867578550" ], | |
| "eventLanes" : [ ], | |
| "services" : [ ] | |
| }, { | |
| "instanceName" : "StreamSelector_03", | |
| "library" : "streamsets-datacollector-basic-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_processor_selector_SelectorDProcessor", | |
| "stageVersion" : "1", | |
| "configuration" : [ { | |
| "name" : "lanePredicates", | |
| "value" : [ { | |
| "outputLane" : "StreamSelector_03OutputLane15000868308540", | |
| "predicate" : "${record:value('/count') > 0}" | |
| }, { | |
| "outputLane" : "StreamSelector_03OutputLane15000868308541", | |
| "predicate" : "default" | |
| } ] | |
| }, { | |
| "name" : "stageOnRecordError", | |
| "value" : "TO_ERROR" | |
| }, { | |
| "name" : "stageRequiredFields", | |
| "value" : [ ] | |
| }, { | |
| "name" : "stageRecordPreconditions", | |
| "value" : [ ] | |
| } ], | |
| "uiInfo" : { | |
| "description" : "", | |
| "label" : "org/company exists?", | |
| "xPos" : 2246.666748046875, | |
| "yPos" : 46.66667175292969, | |
| "stageType" : "PROCESSOR" | |
| }, | |
| "inputLanes" : [ "JDBCLookup_02OutputLane15000867578550" ], | |
| "outputLanes" : [ "StreamSelector_03OutputLane15000868308540", "StreamSelector_03OutputLane15000868308541" ], | |
| "eventLanes" : [ ], | |
| "services" : [ ] | |
| }, { | |
| "instanceName" : "Trash_03", | |
| "library" : "streamsets-datacollector-basic-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_destination_devnull_NullDTarget", | |
| "stageVersion" : "1", | |
| "configuration" : [ ], | |
| "uiInfo" : { | |
| "description" : "", | |
| "label" : "Trash 3", | |
| "xPos" : 2461.666748046875, | |
| "yPos" : 45.00001525878906, | |
| "stageType" : "TARGET" | |
| }, | |
| "inputLanes" : [ "StreamSelector_03OutputLane15000868308540" ], | |
| "outputLanes" : [ ], | |
| "eventLanes" : [ ], | |
| "services" : [ ] | |
| }, { | |
| "instanceName" : "JDBCProducer_04", | |
| "library" : "streamsets-datacollector-jdbc-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_destination_jdbc_JdbcDTarget", | |
| "stageVersion" : "6", | |
| "configuration" : [ { | |
| "name" : "schema", | |
| "value" : "" | |
| }, { | |
| "name" : "tableNameTemplate", | |
| "value" : "org_company" | |
| }, { | |
| "name" : "columnNames", | |
| "value" : [ { | |
| "paramValue" : "?", | |
| "dataType" : "USE_COLUMN_TYPE", | |
| "columnName" : "company", | |
| "field" : "/lookup/data[0]/name" | |
| }, { | |
| "paramValue" : "?", | |
| "dataType" : "USE_COLUMN_TYPE", | |
| "columnName" : "org", | |
| "field" : "/lookup/data[0]/name" | |
| } ] | |
| }, { | |
| "name" : "encloseTableName", | |
| "value" : false | |
| }, { | |
| "name" : "changeLogFormat", | |
| "value" : "NONE" | |
| }, { | |
| "name" : "defaultOperation", | |
| "value" : "INSERT" | |
| }, { | |
| "name" : "unsupportedAction", | |
| "value" : "DISCARD" | |
| }, { | |
| "name" : "useMultiRowInsert", | |
| "value" : false | |
| }, { | |
| "name" : "maxPrepStmtParameters", | |
| "value" : -1 | |
| }, { | |
| "name" : "maxPrepStmtCache", | |
| "value" : -1 | |
| }, { | |
| "name" : "rollbackOnError", | |
| "value" : false | |
| }, { | |
| "name" : "hikariConfigBean.connectionString", | |
| "value" : "${JDBC_URI}?useSSL=false" | |
| }, { | |
| "name" : "hikariConfigBean.useCredentials", | |
| "value" : true | |
| }, { | |
| "name" : "hikariConfigBean.username", | |
| "value" : "${JDBC_USER}" | |
| }, { | |
| "name" : "hikariConfigBean.password", | |
| "value" : "${JDBC_PASSWORD}" | |
| }, { | |
| "name" : "hikariConfigBean.driverProperties", | |
| "value" : [ ] | |
| }, { | |
| "name" : "hikariConfigBean.driverClassName", | |
| "value" : null | |
| }, { | |
| "name" : "hikariConfigBean.connectionTestQuery", | |
| "value" : null | |
| }, { | |
| "name" : "hikariConfigBean.maximumPoolSize", | |
| "value" : 1 | |
| }, { | |
| "name" : "hikariConfigBean.minIdle", | |
| "value" : 1 | |
| }, { | |
| "name" : "hikariConfigBean.connectionTimeout", | |
| "value" : "${30 * SECONDS}" | |
| }, { | |
| "name" : "hikariConfigBean.idleTimeout", | |
| "value" : "${10 * MINUTES}" | |
| }, { | |
| "name" : "hikariConfigBean.maxLifetime", | |
| "value" : "${30 * MINUTES}" | |
| }, { | |
| "name" : "hikariConfigBean.transactionIsolation", | |
| "value" : "DEFAULT" | |
| }, { | |
| "name" : "stageOnRecordError", | |
| "value" : "TO_ERROR" | |
| }, { | |
| "name" : "stageRequiredFields", | |
| "value" : [ ] | |
| }, { | |
| "name" : "stageRecordPreconditions", | |
| "value" : [ ] | |
| }, { | |
| "name" : "hikariConfigBean.initialQuery", | |
| "value" : null | |
| } ], | |
| "uiInfo" : { | |
| "description" : "", | |
| "label" : "org_company table", | |
| "xPos" : 2461.666748046875, | |
| "yPos" : 181.6666717529297, | |
| "stageType" : "TARGET" | |
| }, | |
| "inputLanes" : [ "StreamSelector_03OutputLane15000868308541" ], | |
| "outputLanes" : [ ], | |
| "eventLanes" : [ ], | |
| "services" : [ ] | |
| }, { | |
| "instanceName" : "Trash_05", | |
| "library" : "streamsets-datacollector-basic-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_destination_devnull_NullDTarget", | |
| "stageVersion" : "1", | |
| "configuration" : [ ], | |
| "uiInfo" : { | |
| "description" : "", | |
| "label" : "Trash 5", | |
| "xPos" : 2045, | |
| "yPos" : 186.6666717529297, | |
| "stageType" : "TARGET" | |
| }, | |
| "inputLanes" : [ "RecordDeduplicator_02OutputLane15015108315571" ], | |
| "outputLanes" : [ ], | |
| "eventLanes" : [ ], | |
| "services" : [ ] | |
| }, { | |
| "instanceName" : "HTTPClient_02", | |
| "library" : "streamsets-datacollector-basic-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_processor_http_HttpDProcessor", | |
| "stageVersion" : "11", | |
| "configuration" : [ { | |
| "name" : "conf.dataFormatConfig.compression", | |
| "value" : "NONE" | |
| }, { | |
| "name" : "conf.dataFormatConfig.filePatternInArchive", | |
| "value" : "*" | |
| }, { | |
| "name" : "conf.dataFormatConfig.charset", | |
| "value" : "UTF-8" | |
| }, { | |
| "name" : "conf.dataFormatConfig.removeCtrlChars", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.textMaxLineLen", | |
| "value" : 1024 | |
| }, { | |
| "name" : "conf.dataFormatConfig.useCustomDelimiter", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.customDelimiter", | |
| "value" : "\\r\\n" | |
| }, { | |
| "name" : "conf.dataFormatConfig.includeCustomDelimiterInTheText", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.jsonMaxObjectLen", | |
| "value" : 4096 | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvFileFormat", | |
| "value" : "CSV" | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvHeader", | |
| "value" : "NO_HEADER" | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvAllowExtraColumns", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvExtraColumnPrefix", | |
| "value" : "_extra_" | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvMaxObjectLen", | |
| "value" : 1024 | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvCustomDelimiter", | |
| "value" : "|" | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvCustomEscape", | |
| "value" : "\\" | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvCustomQuote", | |
| "value" : "\"" | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvEnableComments", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvCommentMarker", | |
| "value" : "#" | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvIgnoreEmptyLines", | |
| "value" : true | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvRecordType", | |
| "value" : "LIST_MAP" | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvSkipStartLines", | |
| "value" : 0 | |
| }, { | |
| "name" : "conf.dataFormatConfig.parseNull", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.nullConstant", | |
| "value" : "\\\\N" | |
| }, { | |
| "name" : "conf.dataFormatConfig.xmlRecordElement", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.includeFieldXpathAttributes", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.xPathNamespaceContext", | |
| "value" : [ ] | |
| }, { | |
| "name" : "conf.dataFormatConfig.outputFieldAttributes", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.xmlMaxObjectLen", | |
| "value" : 4096 | |
| }, { | |
| "name" : "conf.dataFormatConfig.logMode", | |
| "value" : "COMMON_LOG_FORMAT" | |
| }, { | |
| "name" : "conf.dataFormatConfig.logMaxObjectLen", | |
| "value" : 1024 | |
| }, { | |
| "name" : "conf.dataFormatConfig.retainOriginalLine", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.customLogFormat", | |
| "value" : "%h %l %u %t \"%r\" %>s %b" | |
| }, { | |
| "name" : "conf.dataFormatConfig.regex", | |
| "value" : "^(\\S+) (\\S+) (\\S+) \\[([\\w:/]+\\s[+\\-]\\d{4})\\] \"(\\S+) (\\S+) (\\S+)\" (\\d{3}) (\\d+)" | |
| }, { | |
| "name" : "conf.dataFormatConfig.fieldPathsToGroupName", | |
| "value" : [ { | |
| "fieldPath" : "/", | |
| "group" : 1 | |
| } ] | |
| }, { | |
| "name" : "conf.dataFormatConfig.grokPatternDefinition", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.grokPattern", | |
| "value" : "%{COMMONAPACHELOG}" | |
| }, { | |
| "name" : "conf.dataFormatConfig.onParseError", | |
| "value" : "ERROR" | |
| }, { | |
| "name" : "conf.dataFormatConfig.maxStackTraceLines", | |
| "value" : 50 | |
| }, { | |
| "name" : "conf.dataFormatConfig.enableLog4jCustomLogFormat", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.log4jCustomLogFormat", | |
| "value" : "%r [%t] %-5p %c %x - %m%n" | |
| }, { | |
| "name" : "conf.dataFormatConfig.avroSchemaSource", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.avroSchema", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.schemaRegistryUrls", | |
| "value" : [ ] | |
| }, { | |
| "name" : "conf.dataFormatConfig.schemaLookupMode", | |
| "value" : "SUBJECT" | |
| }, { | |
| "name" : "conf.dataFormatConfig.subject", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.schemaId", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.protoDescriptorFile", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.messageType", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.isDelimited", | |
| "value" : true | |
| }, { | |
| "name" : "conf.dataFormatConfig.binaryMaxObjectLen", | |
| "value" : 1024 | |
| }, { | |
| "name" : "conf.dataFormatConfig.datagramMode", | |
| "value" : "SYSLOG" | |
| }, { | |
| "name" : "conf.dataFormatConfig.typesDbPath", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.convertTime", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.excludeInterval", | |
| "value" : true | |
| }, { | |
| "name" : "conf.dataFormatConfig.authFilePath", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.netflowOutputValuesMode", | |
| "value" : "RAW_AND_INTERPRETED" | |
| }, { | |
| "name" : "conf.dataFormatConfig.maxTemplateCacheSize", | |
| "value" : -1 | |
| }, { | |
| "name" : "conf.dataFormatConfig.templateCacheTimeoutMs", | |
| "value" : -1 | |
| }, { | |
| "name" : "conf.dataFormatConfig.netflowOutputValuesModeDatagram", | |
| "value" : "RAW_AND_INTERPRETED" | |
| }, { | |
| "name" : "conf.dataFormatConfig.maxTemplateCacheSizeDatagram", | |
| "value" : -1 | |
| }, { | |
| "name" : "conf.dataFormatConfig.templateCacheTimeoutMsDatagram", | |
| "value" : -1 | |
| }, { | |
| "name" : "conf.dataFormatConfig.wholeFileMaxObjectLen", | |
| "value" : 8192 | |
| }, { | |
| "name" : "conf.dataFormatConfig.rateLimit", | |
| "value" : "-1" | |
| }, { | |
| "name" : "conf.dataFormatConfig.verifyChecksum", | |
| "value" : false | |
| }, { | |
| "name" : "conf.outputField", | |
| "value" : "/location" | |
| }, { | |
| "name" : "conf.headerOutputLocation", | |
| "value" : "HEADER" | |
| }, { | |
| "name" : "conf.headerAttributePrefix", | |
| "value" : null | |
| }, { | |
| "name" : "conf.headerOutputField", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormat", | |
| "value" : "JSON" | |
| }, { | |
| "name" : "conf.resourceUrl", | |
| "value" : "https://api.kickfire.com/v2/ip2geo?ip=${record:value('/ip')}&key=fb6d3463b00f3276" | |
| }, { | |
| "name" : "conf.headers", | |
| "value" : [ ] | |
| }, { | |
| "name" : "conf.httpMethod", | |
| "value" : "GET" | |
| }, { | |
| "name" : "conf.methodExpression", | |
| "value" : null | |
| }, { | |
| "name" : "conf.requestBody", | |
| "value" : null | |
| }, { | |
| "name" : "conf.defaultRequestContentType", | |
| "value" : "application/json" | |
| }, { | |
| "name" : "conf.client.transferEncoding", | |
| "value" : "CHUNKED" | |
| }, { | |
| "name" : "conf.client.httpCompression", | |
| "value" : "NONE" | |
| }, { | |
| "name" : "conf.client.connectTimeoutMillis", | |
| "value" : 0 | |
| }, { | |
| "name" : "conf.client.readTimeoutMillis", | |
| "value" : 0 | |
| }, { | |
| "name" : "conf.client.numThreads", | |
| "value" : 1 | |
| }, { | |
| "name" : "conf.client.authType", | |
| "value" : "NONE" | |
| }, { | |
| "name" : "conf.client.useOAuth2", | |
| "value" : false | |
| }, { | |
| "name" : "conf.client.oauth.consumerKey", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.client.oauth.consumerSecret", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.client.oauth.token", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.client.oauth.tokenSecret", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.client.oauth2.credentialsGrantType", | |
| "value" : null | |
| }, { | |
| "name" : "conf.client.oauth2.tokenUrl", | |
| "value" : null | |
| }, { | |
| "name" : "conf.client.oauth2.clientId", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.client.oauth2.clientSecret", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.client.oauth2.username", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.client.oauth2.password", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.client.oauth2.resourceOwnerClientId", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.client.oauth2.resourceOwnerClientSecret", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.client.oauth2.algorithm", | |
| "value" : "NONE" | |
| }, { | |
| "name" : "conf.client.oauth2.key", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.client.oauth2.jwtClaims", | |
| "value" : null | |
| }, { | |
| "name" : "conf.client.oauth2.transferEncoding", | |
| "value" : "BUFFERED" | |
| }, { | |
| "name" : "conf.client.oauth2.additionalValues", | |
| "value" : [ { } ] | |
| }, { | |
| "name" : "conf.client.basicAuth.username", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.client.basicAuth.password", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.client.useProxy", | |
| "value" : false | |
| }, { | |
| "name" : "conf.client.proxy.uri", | |
| "value" : null | |
| }, { | |
| "name" : "conf.client.proxy.username", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.client.proxy.password", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.client.tlsConfig.tlsEnabled", | |
| "value" : false | |
| }, { | |
| "name" : "conf.client.tlsConfig.keyStoreFilePath", | |
| "value" : null | |
| }, { | |
| "name" : "conf.client.tlsConfig.keyStoreType", | |
| "value" : "JKS" | |
| }, { | |
| "name" : "conf.client.tlsConfig.keyStorePassword", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.client.tlsConfig.keyStoreAlgorithm", | |
| "value" : "SunX509" | |
| }, { | |
| "name" : "conf.client.tlsConfig.trustStoreFilePath", | |
| "value" : null | |
| }, { | |
| "name" : "conf.client.tlsConfig.trustStoreType", | |
| "value" : "JKS" | |
| }, { | |
| "name" : "conf.client.tlsConfig.trustStorePassword", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.client.tlsConfig.trustStoreAlgorithm", | |
| "value" : "SunX509" | |
| }, { | |
| "name" : "conf.client.tlsConfig.useDefaultProtocols", | |
| "value" : true | |
| }, { | |
| "name" : "conf.client.tlsConfig.protocols", | |
| "value" : [ ] | |
| }, { | |
| "name" : "conf.client.tlsConfig.useDefaultCiperSuites", | |
| "value" : true | |
| }, { | |
| "name" : "conf.client.tlsConfig.cipherSuites", | |
| "value" : [ ] | |
| }, { | |
| "name" : "conf.client.requestLoggingConfig.enableRequestLogging", | |
| "value" : false | |
| }, { | |
| "name" : "conf.client.requestLoggingConfig.logLevel", | |
| "value" : "FINE" | |
| }, { | |
| "name" : "conf.client.requestLoggingConfig.verbosity", | |
| "value" : "HEADERS_ONLY" | |
| }, { | |
| "name" : "conf.client.requestLoggingConfig.maxEntitySize", | |
| "value" : 0 | |
| }, { | |
| "name" : "conf.rateLimit", | |
| "value" : 3000 | |
| }, { | |
| "name" : "conf.maxRequestCompletionSecs", | |
| "value" : 60 | |
| }, { | |
| "name" : "stageOnRecordError", | |
| "value" : "TO_ERROR" | |
| }, { | |
| "name" : "stageRequiredFields", | |
| "value" : [ ] | |
| }, { | |
| "name" : "stageRecordPreconditions", | |
| "value" : [ ] | |
| }, { | |
| "name" : "conf.dataFormatConfig.excelHeader", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.multiCharacterFieldDelimiter", | |
| "value" : "||" | |
| }, { | |
| "name" : "conf.dataFormatConfig.multiCharacterLineDelimiter", | |
| "value" : "${str:unescapeJava('\\\\n')}" | |
| } ], | |
| "uiInfo" : { | |
| "outputStreamLabels" : null, | |
| "yPos" : -84.0590705871582, | |
| "stageType" : "PROCESSOR", | |
| "rawSource" : null, | |
| "description" : "", | |
| "label" : "Kickfire Location Lookup", | |
| "xPos" : 1811.983154296875 | |
| }, | |
| "inputLanes" : [ "StreamSelector_04OutputLane1500087032682" ], | |
| "outputLanes" : [ "HTTPClient_02OutputLane15269275492590" ], | |
| "eventLanes" : [ ], | |
| "services" : [ ] | |
| }, { | |
| "instanceName" : "ExpressionEvaluator_01", | |
| "library" : "streamsets-datacollector-basic-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_processor_expression_ExpressionDProcessor", | |
| "stageVersion" : "2", | |
| "configuration" : [ { | |
| "name" : "expressionProcessorConfigs", | |
| "value" : [ { | |
| "fieldToSet" : "/location/data/latitude", | |
| "expression" : "${str:length(record:value('/location/data/latitude')) == 0 ? '0' : record:value('/location/data/latitude')}" | |
| }, { | |
| "fieldToSet" : "/location/data/longitude", | |
| "expression" : "${str:length(record:value('/location/data/longitude')) == 0 ? '0' : record:value('/location/data/longitude')}" | |
| }, { | |
| "fieldToSet" : "/lookup/data[0]/naicsCode", | |
| "expression" : "${str:length(record:value('/lookup/data[0]/naicsCode')) == 0 ? '0' : record:value('/lookup/data[0]/naicsCode')}" | |
| } ] | |
| }, { | |
| "name" : "headerAttributeConfigs", | |
| "value" : [ { | |
| "attributeToSet" : "sdc.operation.type", | |
| "headerAttributeExpression" : "${(record:value('/count') > 0) ? '3' : '1'}" | |
| } ] | |
| }, { | |
| "name" : "fieldAttributeConfigs", | |
| "value" : [ { | |
| "fieldToSet" : "/" | |
| } ] | |
| }, { | |
| "name" : "stageOnRecordError", | |
| "value" : "TO_ERROR" | |
| }, { | |
| "name" : "stageRequiredFields", | |
| "value" : [ ] | |
| }, { | |
| "name" : "stageRecordPreconditions", | |
| "value" : [ ] | |
| } ], | |
| "uiInfo" : { | |
| "description" : "", | |
| "label" : "Set Fields, Attributes", | |
| "xPos" : 2041.2034912109375, | |
| "yPos" : -83.14588356018066, | |
| "stageType" : "PROCESSOR" | |
| }, | |
| "inputLanes" : [ "HTTPClient_02OutputLane15269275492590" ], | |
| "outputLanes" : [ "ExpressionEvaluator_01OutputLane15103392592010" ], | |
| "eventLanes" : [ ], | |
| "services" : [ ] | |
| }, { | |
| "instanceName" : "FieldTypeConverter_02", | |
| "library" : "streamsets-datacollector-basic-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_processor_fieldtypeconverter_FieldTypeConverterDProcessor", | |
| "stageVersion" : "2", | |
| "configuration" : [ { | |
| "name" : "convertBy", | |
| "value" : "BY_FIELD" | |
| }, { | |
| "name" : "fieldTypeConverterConfigs", | |
| "value" : [ { | |
| "fields" : [ "/location/data/latitude", "/location/data/longitude" ], | |
| "targetType" : "FLOAT", | |
| "treatInputFieldAsDate" : false, | |
| "dataLocale" : "en,US", | |
| "scale" : -1, | |
| "decimalScaleRoundingStrategy" : "ROUND_UNNECESSARY", | |
| "dateFormat" : "YYYY_MM_DD", | |
| "zonedDateTimeFormat" : "ISO_ZONED_DATE_TIME", | |
| "encoding" : "UTF-8" | |
| }, { | |
| "fields" : [ "/lookup/data[0]/naicsCode" ], | |
| "targetType" : "INTEGER", | |
| "treatInputFieldAsDate" : false, | |
| "dataLocale" : "en,US", | |
| "scale" : -1, | |
| "decimalScaleRoundingStrategy" : "ROUND_UNNECESSARY", | |
| "dateFormat" : "YYYY_MM_DD", | |
| "zonedDateTimeFormat" : "ISO_ZONED_DATE_TIME", | |
| "encoding" : "UTF-8" | |
| } ] | |
| }, { | |
| "name" : "wholeTypeConverterConfigs", | |
| "value" : [ { | |
| "sourceType" : "INTEGER", | |
| "targetType" : "INTEGER", | |
| "treatInputFieldAsDate" : false, | |
| "dataLocale" : "en,US", | |
| "scale" : -1, | |
| "decimalScaleRoundingStrategy" : "ROUND_UNNECESSARY", | |
| "dateFormat" : "YYYY_MM_DD", | |
| "zonedDateTimeFormat" : "ISO_ZONED_DATE_TIME", | |
| "encoding" : "UTF-8" | |
| } ] | |
| }, { | |
| "name" : "stageOnRecordError", | |
| "value" : "TO_ERROR" | |
| }, { | |
| "name" : "stageRequiredFields", | |
| "value" : [ ] | |
| }, { | |
| "name" : "stageRecordPreconditions", | |
| "value" : [ ] | |
| } ], | |
| "uiInfo" : { | |
| "outputStreamLabels" : null, | |
| "yPos" : -86.61127853393555, | |
| "stageType" : "PROCESSOR", | |
| "rawSource" : null, | |
| "description" : "", | |
| "label" : "Convert Field Types", | |
| "xPos" : 2247.254638671875 | |
| }, | |
| "inputLanes" : [ "ExpressionEvaluator_01OutputLane15103392592010" ], | |
| "outputLanes" : [ "FieldTypeConverter_02OutputLane15371495385510" ], | |
| "eventLanes" : [ ], | |
| "services" : [ ] | |
| }, { | |
| "instanceName" : "JDBCProducer_03", | |
| "library" : "streamsets-datacollector-jdbc-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_destination_jdbc_JdbcDTarget", | |
| "stageVersion" : "6", | |
| "configuration" : [ { | |
| "name" : "schema", | |
| "value" : "" | |
| }, { | |
| "name" : "tableNameTemplate", | |
| "value" : "client" | |
| }, { | |
| "name" : "columnNames", | |
| "value" : [ { | |
| "paramValue" : "?", | |
| "dataType" : "USE_COLUMN_TYPE", | |
| "columnName" : "ip", | |
| "field" : "/ip" | |
| }, { | |
| "paramValue" : "?", | |
| "dataType" : "USE_COLUMN_TYPE", | |
| "columnName" : "city", | |
| "field" : "/location/data/city" | |
| }, { | |
| "paramValue" : "?", | |
| "dataType" : "USE_COLUMN_TYPE", | |
| "columnName" : "region", | |
| "field" : "/location/data/region" | |
| }, { | |
| "paramValue" : "?", | |
| "dataType" : "USE_COLUMN_TYPE", | |
| "columnName" : "countryShort", | |
| "field" : "/location/data/countryShort" | |
| }, { | |
| "paramValue" : "?", | |
| "columnName" : "latitude", | |
| "dataType" : "USE_COLUMN_TYPE", | |
| "field" : "/location/data/latitude" | |
| }, { | |
| "paramValue" : "?", | |
| "dataType" : "USE_COLUMN_TYPE", | |
| "columnName" : "longitude", | |
| "field" : "/location/data/longitude" | |
| }, { | |
| "paramValue" : "?", | |
| "dataType" : "USE_COLUMN_TYPE", | |
| "columnName" : "org", | |
| "field" : "/lookup/data[0]/name" | |
| }, { | |
| "paramValue" : "?", | |
| "dataType" : "USE_COLUMN_TYPE", | |
| "columnName" : "website", | |
| "field" : "/lookup/data[0]/website" | |
| }, { | |
| "paramValue" : "?", | |
| "dataType" : "USE_COLUMN_TYPE", | |
| "columnName" : "regionShort", | |
| "field" : "/location/data/regionShort" | |
| }, { | |
| "paramValue" : "?", | |
| "columnName" : "country", | |
| "field" : "/location/data/country", | |
| "dataType" : "USE_COLUMN_TYPE" | |
| }, { | |
| "paramValue" : "?", | |
| "dataType" : "USE_COLUMN_TYPE", | |
| "columnName" : "employees", | |
| "field" : "/lookup/data[0]/employees" | |
| }, { | |
| "paramValue" : "?", | |
| "dataType" : "USE_COLUMN_TYPE", | |
| "columnName" : "revenue", | |
| "field" : "/lookup/data[0]/revenue" | |
| }, { | |
| "paramValue" : "?", | |
| "dataType" : "USE_COLUMN_TYPE", | |
| "columnName" : "sicGroup", | |
| "field" : "/lookup/data[0]/sicGroup" | |
| }, { | |
| "paramValue" : "?", | |
| "dataType" : "USE_COLUMN_TYPE", | |
| "columnName" : "category", | |
| "field" : "/lookup/data[0]/category" | |
| }, { | |
| "paramValue" : "?", | |
| "dataType" : "USE_COLUMN_TYPE", | |
| "columnName" : "sicCode", | |
| "field" : "/lookup/data[0]/sicCode" | |
| }, { | |
| "paramValue" : "?", | |
| "dataType" : "USE_COLUMN_TYPE", | |
| "columnName" : "naicsGroup", | |
| "field" : "/lookup/data[0]/naicsGroup" | |
| }, { | |
| "paramValue" : "?", | |
| "dataType" : "USE_COLUMN_TYPE", | |
| "columnName" : "category2", | |
| "field" : "/lookup/data[0]/category2" | |
| }, { | |
| "paramValue" : "?", | |
| "dataType" : "USE_COLUMN_TYPE", | |
| "columnName" : "naicsCode", | |
| "field" : "/lookup/data[0]/naicsCode" | |
| }, { | |
| "paramValue" : "?", | |
| "dataType" : "USE_COLUMN_TYPE", | |
| "columnName" : "stockSymbol", | |
| "field" : "/lookup/data[0]/stockSymbol" | |
| }, { | |
| "paramValue" : "?", | |
| "dataType" : "USE_COLUMN_TYPE", | |
| "columnName" : "facebook", | |
| "field" : "/lookup/data[0]/facebook" | |
| }, { | |
| "paramValue" : "?", | |
| "dataType" : "USE_COLUMN_TYPE", | |
| "columnName" : "twitter", | |
| "field" : "/lookup/data[0]/twitter" | |
| }, { | |
| "paramValue" : "?", | |
| "dataType" : "USE_COLUMN_TYPE", | |
| "columnName" : "linkedIn", | |
| "field" : "/lookup/data[0]/linkedIn" | |
| }, { | |
| "paramValue" : "?", | |
| "dataType" : "USE_COLUMN_TYPE", | |
| "columnName" : "confidence", | |
| "field" : "/lookup/data[0]/confidence" | |
| }, { | |
| "paramValue" : "?", | |
| "dataType" : "USE_COLUMN_TYPE", | |
| "columnName" : "isISP", | |
| "field" : "/lookup/data[0]/isISP" | |
| }, { | |
| "paramValue" : "?", | |
| "dataType" : "USE_COLUMN_TYPE", | |
| "columnName" : "isWifi", | |
| "field" : "/lookup/data[0]/isWifi" | |
| } ] | |
| }, { | |
| "name" : "encloseTableName", | |
| "value" : false | |
| }, { | |
| "name" : "changeLogFormat", | |
| "value" : "NONE" | |
| }, { | |
| "name" : "defaultOperation", | |
| "value" : "INSERT" | |
| }, { | |
| "name" : "unsupportedAction", | |
| "value" : "DISCARD" | |
| }, { | |
| "name" : "useMultiRowInsert", | |
| "value" : false | |
| }, { | |
| "name" : "maxPrepStmtParameters", | |
| "value" : -1 | |
| }, { | |
| "name" : "maxPrepStmtCache", | |
| "value" : -1 | |
| }, { | |
| "name" : "rollbackOnError", | |
| "value" : false | |
| }, { | |
| "name" : "hikariConfigBean.connectionString", | |
| "value" : "${JDBC_URI}?useSSL=false" | |
| }, { | |
| "name" : "hikariConfigBean.useCredentials", | |
| "value" : true | |
| }, { | |
| "name" : "hikariConfigBean.username", | |
| "value" : "${JDBC_USER}" | |
| }, { | |
| "name" : "hikariConfigBean.password", | |
| "value" : "${JDBC_PASSWORD}" | |
| }, { | |
| "name" : "hikariConfigBean.driverProperties", | |
| "value" : [ ] | |
| }, { | |
| "name" : "hikariConfigBean.driverClassName", | |
| "value" : null | |
| }, { | |
| "name" : "hikariConfigBean.connectionTestQuery", | |
| "value" : null | |
| }, { | |
| "name" : "hikariConfigBean.maximumPoolSize", | |
| "value" : 1 | |
| }, { | |
| "name" : "hikariConfigBean.minIdle", | |
| "value" : 1 | |
| }, { | |
| "name" : "hikariConfigBean.connectionTimeout", | |
| "value" : "${30 * SECONDS}" | |
| }, { | |
| "name" : "hikariConfigBean.idleTimeout", | |
| "value" : "${10 * MINUTES}" | |
| }, { | |
| "name" : "hikariConfigBean.maxLifetime", | |
| "value" : "${30 * MINUTES}" | |
| }, { | |
| "name" : "hikariConfigBean.transactionIsolation", | |
| "value" : "DEFAULT" | |
| }, { | |
| "name" : "stageOnRecordError", | |
| "value" : "TO_ERROR" | |
| }, { | |
| "name" : "stageRequiredFields", | |
| "value" : [ ] | |
| }, { | |
| "name" : "stageRecordPreconditions", | |
| "value" : [ ] | |
| }, { | |
| "name" : "hikariConfigBean.initialQuery", | |
| "value" : null | |
| } ], | |
| "uiInfo" : { | |
| "description" : "", | |
| "label" : "client table", | |
| "xPos" : 2462.254638671875, | |
| "yPos" : -85.71685791015625, | |
| "stageType" : "TARGET" | |
| }, | |
| "inputLanes" : [ "FieldTypeConverter_02OutputLane15371495385510" ], | |
| "outputLanes" : [ ], | |
| "eventLanes" : [ ], | |
| "services" : [ ] | |
| } ], | |
| "errorStage" : { | |
| "instanceName" : "WritetoFile_ErrorStage", | |
| "library" : "streamsets-datacollector-basic-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_destination_recordstolocalfilesystem_ToErrorLocalFSDTarget", | |
| "stageVersion" : "1", | |
| "configuration" : [ { | |
| "name" : "directory", | |
| "value" : "/tmp" | |
| }, { | |
| "name" : "uniquePrefix", | |
| "value" : "sdc-errors-${sdc:id()}" | |
| }, { | |
| "name" : "rotationIntervalSecs", | |
| "value" : "${1 * HOURS}" | |
| }, { | |
| "name" : "maxFileSizeMbs", | |
| "value" : 512 | |
| } ], | |
| "uiInfo" : { | |
| "description" : "", | |
| "label" : "Error Records - Write to File", | |
| "xPos" : 280, | |
| "yPos" : 50, | |
| "stageType" : "TARGET" | |
| }, | |
| "inputLanes" : [ ], | |
| "outputLanes" : [ ], | |
| "eventLanes" : [ ], | |
| "services" : [ ] | |
| }, | |
| "info" : { | |
| "pipelineId" : "Metrics635c20ed-872c-48d7-87fb-7043ab937ae6", | |
| "title" : "Metrics", | |
| "description" : "", | |
| "created" : 1552079218056, | |
| "lastModified" : 1556147969721, | |
| "creator" : "admin", | |
| "lastModifier" : "admin", | |
| "lastRev" : "0", | |
| "uuid" : "08bce59b-151b-47d4-97dc-5d4f58da9672", | |
| "valid" : true, | |
| "metadata" : { | |
| "labels" : [ ], | |
| "dpm.pipeline.id" : "ad4b15f6-787b-486d-8f8e-6d0ab2f8be2d:dw.streamsets.com", | |
| "dpm.pipeline.version" : "9", | |
| "dpm.pipeline.commit.id" : "704222da-4104-47bf-ba60-ca391b7b1dd1:dw.streamsets.com", | |
| "dpm.pipeline.rules.id" : "778faa86-f30a-4b4f-9b0a-4666d1e54bc1:dw.streamsets.com", | |
| "dpm.base.url" : "https://cloud.streamsets.com" | |
| }, | |
| "name" : "Metrics635c20ed-872c-48d7-87fb-7043ab937ae6", | |
| "sdcVersion" : "3.8.0", | |
| "sdcId" : "2e5c18f6-2e5b-11e9-a456-3ff9fba8972e" | |
| }, | |
| "metadata" : { | |
| "labels" : [ ], | |
| "dpm.pipeline.id" : "ad4b15f6-787b-486d-8f8e-6d0ab2f8be2d:dw.streamsets.com", | |
| "dpm.pipeline.version" : "9", | |
| "dpm.pipeline.commit.id" : "704222da-4104-47bf-ba60-ca391b7b1dd1:dw.streamsets.com", | |
| "dpm.pipeline.rules.id" : "778faa86-f30a-4b4f-9b0a-4666d1e54bc1:dw.streamsets.com", | |
| "dpm.base.url" : "https://cloud.streamsets.com" | |
| }, | |
| "statsAggregatorStage" : { | |
| "instanceName" : "WritetoDPMdirectly_StatsAggregatorStage", | |
| "library" : "streamsets-datacollector-basic-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_destination_devnull_StatsDpmDirectlyDTarget", | |
| "stageVersion" : "1", | |
| "configuration" : [ ], | |
| "uiInfo" : { | |
| "description" : "", | |
| "label" : "Stats Aggregator - Write to DPM directly", | |
| "xPos" : 268, | |
| "yPos" : 50, | |
| "stageType" : "TARGET" | |
| }, | |
| "inputLanes" : [ ], | |
| "outputLanes" : [ ], | |
| "eventLanes" : [ ], | |
| "services" : [ ] | |
| }, | |
| "startEventStages" : [ { | |
| "instanceName" : "Discard_StartEventStage", | |
| "library" : "streamsets-datacollector-basic-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget", | |
| "stageVersion" : "1", | |
| "configuration" : [ ], | |
| "uiInfo" : { | |
| "outputStreamLabels" : null, | |
| "yPos" : 50, | |
| "stageType" : "TARGET", | |
| "rawSource" : null, | |
| "description" : "", | |
| "label" : "Start Event - Discard", | |
| "xPos" : 2257.4716796875 | |
| }, | |
| "inputLanes" : [ ], | |
| "outputLanes" : [ ], | |
| "eventLanes" : [ ], | |
| "services" : [ ] | |
| } ], | |
| "stopEventStages" : [ { | |
| "instanceName" : "Discard_StopEventStage", | |
| "library" : "streamsets-datacollector-basic-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget", | |
| "stageVersion" : "1", | |
| "configuration" : [ ], | |
| "uiInfo" : { | |
| "outputStreamLabels" : null, | |
| "yPos" : 50, | |
| "stageType" : "TARGET", | |
| "rawSource" : null, | |
| "description" : "", | |
| "label" : "Stop Event - Discard", | |
| "xPos" : 2257.4716796875 | |
| }, | |
| "inputLanes" : [ ], | |
| "outputLanes" : [ ], | |
| "eventLanes" : [ ], | |
| "services" : [ ] | |
| } ], | |
| "testOriginStage" : { | |
| "instanceName" : "DevRawDataSource_TestOriginStage", | |
| "library" : "streamsets-datacollector-dev-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_devtest_rawdata_RawDataDSource", | |
| "stageVersion" : "3", | |
| "configuration" : [ { | |
| "name" : "rawData", | |
| "value" : "{\n \"f1\": \"abc\",\n \"f2\": \"xyz\",\n \"f3\": \"lmn\"\n}" | |
| }, { | |
| "name" : "stopAfterFirstBatch", | |
| "value" : false | |
| }, { | |
| "name" : "stageOnRecordError", | |
| "value" : "TO_ERROR" | |
| } ], | |
| "uiInfo" : { | |
| "outputStreamLabels" : null, | |
| "yPos" : 50, | |
| "stageType" : "SOURCE", | |
| "rawSource" : null, | |
| "description" : "", | |
| "label" : "Test Origin - Dev Raw Data Source", | |
| "xPos" : 2682.19677734375 | |
| }, | |
| "inputLanes" : [ ], | |
| "outputLanes" : [ "DevRawDataSource_TestOriginStageOutputLane15380581013780" ], | |
| "eventLanes" : [ ], | |
| "services" : [ { | |
| "service" : "com.streamsets.pipeline.api.service.dataformats.DataFormatParserService", | |
| "serviceVersion" : 1, | |
| "configuration" : [ { | |
| "name" : "displayFormats", | |
| "value" : "DELIMITED,JSON,LOG,SDC_JSON,TEXT,XML" | |
| }, { | |
| "name" : "dataFormat", | |
| "value" : "JSON" | |
| }, { | |
| "name" : "dataFormatConfig.compression", | |
| "value" : "NONE" | |
| }, { | |
| "name" : "dataFormatConfig.filePatternInArchive", | |
| "value" : "*" | |
| }, { | |
| "name" : "dataFormatConfig.charset", | |
| "value" : "UTF-8" | |
| }, { | |
| "name" : "dataFormatConfig.removeCtrlChars", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.textMaxLineLen", | |
| "value" : 1024 | |
| }, { | |
| "name" : "dataFormatConfig.useCustomDelimiter", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.customDelimiter", | |
| "value" : "\\r\\n" | |
| }, { | |
| "name" : "dataFormatConfig.includeCustomDelimiterInTheText", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.jsonContent", | |
| "value" : "MULTIPLE_OBJECTS" | |
| }, { | |
| "name" : "dataFormatConfig.jsonMaxObjectLen", | |
| "value" : 4096 | |
| }, { | |
| "name" : "dataFormatConfig.csvFileFormat", | |
| "value" : "CSV" | |
| }, { | |
| "name" : "dataFormatConfig.csvHeader", | |
| "value" : "NO_HEADER" | |
| }, { | |
| "name" : "dataFormatConfig.csvAllowExtraColumns", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.csvExtraColumnPrefix", | |
| "value" : "_extra_" | |
| }, { | |
| "name" : "dataFormatConfig.csvMaxObjectLen", | |
| "value" : 1024 | |
| }, { | |
| "name" : "dataFormatConfig.csvCustomDelimiter", | |
| "value" : "|" | |
| }, { | |
| "name" : "dataFormatConfig.csvCustomEscape", | |
| "value" : "\\" | |
| }, { | |
| "name" : "dataFormatConfig.csvCustomQuote", | |
| "value" : "\"" | |
| }, { | |
| "name" : "dataFormatConfig.csvEnableComments", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.csvCommentMarker", | |
| "value" : "#" | |
| }, { | |
| "name" : "dataFormatConfig.csvIgnoreEmptyLines", | |
| "value" : true | |
| }, { | |
| "name" : "dataFormatConfig.csvRecordType", | |
| "value" : "LIST_MAP" | |
| }, { | |
| "name" : "dataFormatConfig.csvSkipStartLines", | |
| "value" : 0 | |
| }, { | |
| "name" : "dataFormatConfig.parseNull", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.nullConstant", | |
| "value" : "\\\\N" | |
| }, { | |
| "name" : "dataFormatConfig.xmlRecordElement", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.includeFieldXpathAttributes", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.xPathNamespaceContext", | |
| "value" : [ ] | |
| }, { | |
| "name" : "dataFormatConfig.outputFieldAttributes", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.xmlMaxObjectLen", | |
| "value" : 4096 | |
| }, { | |
| "name" : "dataFormatConfig.logMode", | |
| "value" : "COMMON_LOG_FORMAT" | |
| }, { | |
| "name" : "dataFormatConfig.logMaxObjectLen", | |
| "value" : 1024 | |
| }, { | |
| "name" : "dataFormatConfig.retainOriginalLine", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.customLogFormat", | |
| "value" : "%h %l %u %t \"%r\" %>s %b" | |
| }, { | |
| "name" : "dataFormatConfig.regex", | |
| "value" : "^(\\S+) (\\S+) (\\S+) \\[([\\w:/]+\\s[+\\-]\\d{4})\\] \"(\\S+) (\\S+) (\\S+)\" (\\d{3}) (\\d+)" | |
| }, { | |
| "name" : "dataFormatConfig.fieldPathsToGroupName", | |
| "value" : [ { | |
| "fieldPath" : "/", | |
| "group" : 1 | |
| } ] | |
| }, { | |
| "name" : "dataFormatConfig.grokPatternDefinition", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.grokPattern", | |
| "value" : "%{COMMONAPACHELOG}" | |
| }, { | |
| "name" : "dataFormatConfig.onParseError", | |
| "value" : "ERROR" | |
| }, { | |
| "name" : "dataFormatConfig.maxStackTraceLines", | |
| "value" : 50 | |
| }, { | |
| "name" : "dataFormatConfig.enableLog4jCustomLogFormat", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.log4jCustomLogFormat", | |
| "value" : "%r [%t] %-5p %c %x - %m%n" | |
| }, { | |
| "name" : "dataFormatConfig.avroSchemaSource", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.avroSchema", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.schemaRegistryUrls", | |
| "value" : [ ] | |
| }, { | |
| "name" : "dataFormatConfig.schemaLookupMode", | |
| "value" : "SUBJECT" | |
| }, { | |
| "name" : "dataFormatConfig.subject", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.schemaId", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.protoDescriptorFile", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.messageType", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.isDelimited", | |
| "value" : true | |
| }, { | |
| "name" : "dataFormatConfig.binaryMaxObjectLen", | |
| "value" : 1024 | |
| }, { | |
| "name" : "dataFormatConfig.datagramMode", | |
| "value" : "SYSLOG" | |
| }, { | |
| "name" : "dataFormatConfig.typesDbPath", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.convertTime", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.excludeInterval", | |
| "value" : true | |
| }, { | |
| "name" : "dataFormatConfig.authFilePath", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.netflowOutputValuesMode", | |
| "value" : "RAW_AND_INTERPRETED" | |
| }, { | |
| "name" : "dataFormatConfig.maxTemplateCacheSize", | |
| "value" : -1 | |
| }, { | |
| "name" : "dataFormatConfig.templateCacheTimeoutMs", | |
| "value" : -1 | |
| }, { | |
| "name" : "dataFormatConfig.netflowOutputValuesModeDatagram", | |
| "value" : "RAW_AND_INTERPRETED" | |
| }, { | |
| "name" : "dataFormatConfig.maxTemplateCacheSizeDatagram", | |
| "value" : -1 | |
| }, { | |
| "name" : "dataFormatConfig.templateCacheTimeoutMsDatagram", | |
| "value" : -1 | |
| }, { | |
| "name" : "dataFormatConfig.wholeFileMaxObjectLen", | |
| "value" : 8192 | |
| }, { | |
| "name" : "dataFormatConfig.rateLimit", | |
| "value" : "-1" | |
| }, { | |
| "name" : "dataFormatConfig.verifyChecksum", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.excelHeader", | |
| "value" : null | |
| } ] | |
| } ] | |
| }, | |
| "valid" : true, | |
| "issues" : { | |
| "stageIssues" : { }, | |
| "pipelineIssues" : [ ], | |
| "issueCount" : 0 | |
| }, | |
| "previewable" : true | |
| }, | |
| "pipelineRules" : { | |
| "schemaVersion" : 3, | |
| "version" : 2, | |
| "metricsRuleDefinitions" : [ { | |
| "id" : "badRecordsAlertID", | |
| "alertText" : "High incidence of Error Records", | |
| "metricId" : "pipeline.batchErrorRecords.counter", | |
| "metricType" : "COUNTER", | |
| "metricElement" : "COUNTER_COUNT", | |
| "condition" : "${value() > 100}", | |
| "sendEmail" : false, | |
| "enabled" : false, | |
| "timestamp" : 1497456417919, | |
| "valid" : true | |
| }, { | |
| "id" : "stageErrorAlertID", | |
| "alertText" : "High incidence of Stage Errors", | |
| "metricId" : "pipeline.batchErrorMessages.counter", | |
| "metricType" : "COUNTER", | |
| "metricElement" : "COUNTER_COUNT", | |
| "condition" : "${value() > 100}", | |
| "sendEmail" : false, | |
| "enabled" : false, | |
| "timestamp" : 1497456417919, | |
| "valid" : true | |
| }, { | |
| "id" : "idleGaugeID", | |
| "alertText" : "Pipeline is Idle", | |
| "metricId" : "RuntimeStatsGauge.gauge", | |
| "metricType" : "GAUGE", | |
| "metricElement" : "TIME_OF_LAST_RECEIVED_RECORD", | |
| "condition" : "${time:now() - value() > 120000}", | |
| "sendEmail" : false, | |
| "enabled" : false, | |
| "timestamp" : 1497456417919, | |
| "valid" : true | |
| }, { | |
| "id" : "batchTimeAlertID", | |
| "alertText" : "Batch taking more time to process", | |
| "metricId" : "RuntimeStatsGauge.gauge", | |
| "metricType" : "GAUGE", | |
| "metricElement" : "CURRENT_BATCH_AGE", | |
| "condition" : "${value() > 200}", | |
| "sendEmail" : false, | |
| "enabled" : false, | |
| "timestamp" : 1497456417919, | |
| "valid" : true | |
| }, { | |
| "id" : "memoryLimitAlertID", | |
| "alertText" : "Memory limit for pipeline exceeded", | |
| "metricId" : "pipeline.memoryConsumed.counter", | |
| "metricType" : "COUNTER", | |
| "metricElement" : "COUNTER_COUNT", | |
| "condition" : "${value() > (jvm:maxMemoryMB() * 0.65)}", | |
| "sendEmail" : false, | |
| "enabled" : false, | |
| "timestamp" : 1497456417919, | |
| "valid" : true | |
| } ], | |
| "dataRuleDefinitions" : [ ], | |
| "driftRuleDefinitions" : [ ], | |
| "uuid" : "4bff984a-4bed-495a-817b-5d1019538a06", | |
| "configuration" : [ { | |
| "name" : "webhookConfigs", | |
| "value" : [ ] | |
| }, { | |
| "name" : "emailIDs", | |
| "value" : [ ] | |
| } ], | |
| "ruleIssues" : [ ], | |
| "configIssues" : [ ] | |
| }, | |
| "libraryDefinitions" : null | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment