Created
December 7, 2019 12:31
-
-
Save Jammink2/23c548aeb6617c3528697e8526f95c18 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "pipelineConfig" : { | |
| "schemaVersion" : 6, | |
| "version" : 16, | |
| "pipelineId" : "LogsfromSSDCtoElasticsearch42654be3-3013-4bc0-b981-75e066aec41c", | |
| "title" : "Logs_from_SSDC_to_Elasticsearch_and_Cassandra", | |
| "description" : "Pipeline from tutorial 1", | |
| "uuid" : "cbbbfff6-4a1a-4548-b623-3f452c816502", | |
| "configuration" : [ { | |
| "name" : "executionMode", | |
| "value" : "STANDALONE" | |
| }, { | |
| "name" : "edgeHttpUrl", | |
| "value" : "http://localhost:18633" | |
| }, { | |
| "name" : "deliveryGuarantee", | |
| "value" : "AT_LEAST_ONCE" | |
| }, { | |
| "name" : "testOriginStage", | |
| "value" : "streamsets-datacollector-dev-lib::com_streamsets_pipeline_stage_devtest_rawdata_RawDataDSource::3" | |
| }, { | |
| "name" : "startEventStage", | |
| "value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1" | |
| }, { | |
| "name" : "stopEventStage", | |
| "value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1" | |
| }, { | |
| "name" : "shouldRetry", | |
| "value" : true | |
| }, { | |
| "name" : "triggerInterval", | |
| "value" : 2000 | |
| }, { | |
| "name" : "retryAttempts", | |
| "value" : -1 | |
| }, { | |
| "name" : "ludicrousMode", | |
| "value" : false | |
| }, { | |
| "name" : "ludicrousModeInputCount", | |
| "value" : false | |
| }, { | |
| "name" : "advancedErrorHandling", | |
| "value" : false | |
| }, { | |
| "name" : "notifyOnStates", | |
| "value" : [ "RUN_ERROR", "STOPPED", "FINISHED" ] | |
| }, { | |
| "name" : "emailIDs", | |
| "value" : [ ] | |
| }, { | |
| "name" : "constants", | |
| "value" : [ ] | |
| }, { | |
| "name" : "badRecordsHandling", | |
| "value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1" | |
| }, { | |
| "name" : "errorRecordPolicy", | |
| "value" : "ORIGINAL_RECORD" | |
| }, { | |
| "name" : "statsAggregatorStage", | |
| "value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_StatsDpmDirectlyDTarget::1" | |
| }, { | |
| "name" : "workerCount", | |
| "value" : 0 | |
| }, { | |
| "name" : "clusterSlaveMemory", | |
| "value" : 2048 | |
| }, { | |
| "name" : "clusterSlaveJavaOpts", | |
| "value" : "-XX:+UseConcMarkSweepGC -XX:+UseParNewGC -Dlog4j.debug" | |
| }, { | |
| "name" : "clusterLauncherEnv", | |
| "value" : [ ] | |
| }, { | |
| "name" : "mesosDispatcherURL", | |
| "value" : null | |
| }, { | |
| "name" : "logLevel", | |
| "value" : "INFO" | |
| }, { | |
| "name" : "hdfsS3ConfDir", | |
| "value" : null | |
| }, { | |
| "name" : "rateLimit", | |
| "value" : 0 | |
| }, { | |
| "name" : "maxRunners", | |
| "value" : 0 | |
| }, { | |
| "name" : "shouldCreateFailureSnapshot", | |
| "value" : true | |
| }, { | |
| "name" : "runnerIdleTIme", | |
| "value" : 60 | |
| }, { | |
| "name" : "webhookConfigs", | |
| "value" : [ ] | |
| }, { | |
| "name" : "sparkConfigs", | |
| "value" : [ ] | |
| }, { | |
| "name" : "preprocessScript", | |
| "value" : "/*\nThe following script define a method\nthat increments an integer by 1 \nand registers it as a UDF with \nthe SparkSession, which can be accessed\nusing the variable named \"spark\":\ndef inc(i: Integer): Integer = {\n i + 1\n}\nspark.udf.register (\"inc\", inc _)\n\n*/" | |
| }, { | |
| "name" : "clusterConfig.clusterType", | |
| "value" : "LOCAL" | |
| }, { | |
| "name" : "clusterConfig.sparkMasterUrl", | |
| "value" : "local[*]" | |
| }, { | |
| "name" : "clusterConfig.deployMode", | |
| "value" : "CLIENT" | |
| }, { | |
| "name" : "clusterConfig.hadoopUserName", | |
| "value" : null | |
| }, { | |
| "name" : "clusterConfig.sparkAppName", | |
| "value" : "${pipeline:title()}" | |
| }, { | |
| "name" : "clusterConfig.stagingDir", | |
| "value" : "/streamsets" | |
| }, { | |
| "name" : "clusterConfig.useYarnKerberosKeytab", | |
| "value" : false | |
| }, { | |
| "name" : "clusterConfig.yarnKerberosKeytabSource", | |
| "value" : "PROPERTIES_FILE" | |
| }, { | |
| "name" : "clusterConfig.yarnKerberosKeytab", | |
| "value" : null | |
| }, { | |
| "name" : "clusterConfig.yarnKerberosPrincipal", | |
| "value" : "name@DOMAIN" | |
| }, { | |
| "name" : "databricksConfig.baseUrl", | |
| "value" : null | |
| }, { | |
| "name" : "databricksConfig.credentialType", | |
| "value" : null | |
| }, { | |
| "name" : "databricksConfig.username", | |
| "value" : "" | |
| }, { | |
| "name" : "databricksConfig.password", | |
| "value" : "" | |
| }, { | |
| "name" : "databricksConfig.token", | |
| "value" : "" | |
| }, { | |
| "name" : "databricksConfig.provisionNewCluster", | |
| "value" : true | |
| }, { | |
| "name" : "databricksConfig.clusterId", | |
| "value" : null | |
| }, { | |
| "name" : "databricksConfig.clusterConfig", | |
| "value" : "{\n \"num_workers\": 8,\n \"spark_version\": \"5.3.x-scala2.11\",\n \"node_type_id\": \"i3.xlarge\"\n}" | |
| }, { | |
| "name" : "databricksConfig.terminateCluster", | |
| "value" : false | |
| }, { | |
| "name" : "livyConfig.baseUrl", | |
| "value" : "https://localhost:30443/gateway/default/livy/v1/" | |
| }, { | |
| "name" : "livyConfig.username", | |
| "value" : "" | |
| }, { | |
| "name" : "livyConfig.password", | |
| "value" : "" | |
| }, { | |
| "name" : "amazonEMRConfig.userRegion", | |
| "value" : null | |
| }, { | |
| "name" : "amazonEMRConfig.userRegionCustom", | |
| "value" : null | |
| }, { | |
| "name" : "amazonEMRConfig.accessKey", | |
| "value" : "" | |
| }, { | |
| "name" : "amazonEMRConfig.secretKey", | |
| "value" : "" | |
| }, { | |
| "name" : "amazonEMRConfig.s3StagingUri", | |
| "value" : null | |
| }, { | |
| "name" : "amazonEMRConfig.provisionNewCluster", | |
| "value" : false | |
| }, { | |
| "name" : "amazonEMRConfig.clusterId", | |
| "value" : null | |
| }, { | |
| "name" : "amazonEMRConfig.clusterPrefix", | |
| "value" : null | |
| }, { | |
| "name" : "amazonEMRConfig.terminateCluster", | |
| "value" : false | |
| }, { | |
| "name" : "amazonEMRConfig.loggingEnabled", | |
| "value" : true | |
| }, { | |
| "name" : "amazonEMRConfig.s3LogUri", | |
| "value" : null | |
| }, { | |
| "name" : "amazonEMRConfig.enableEMRDebugging", | |
| "value" : true | |
| }, { | |
| "name" : "amazonEMRConfig.serviceRole", | |
| "value" : "EMR_DefaultRole" | |
| }, { | |
| "name" : "amazonEMRConfig.jobFlowRole", | |
| "value" : "EMR_EC2_DefaultRole" | |
| }, { | |
| "name" : "amazonEMRConfig.visibleToAllUsers", | |
| "value" : true | |
| }, { | |
| "name" : "amazonEMRConfig.ec2SubnetId", | |
| "value" : null | |
| }, { | |
| "name" : "amazonEMRConfig.masterSecurityGroup", | |
| "value" : null | |
| }, { | |
| "name" : "amazonEMRConfig.slaveSecurityGroup", | |
| "value" : null | |
| }, { | |
| "name" : "amazonEMRConfig.instanceCount", | |
| "value" : 2 | |
| }, { | |
| "name" : "amazonEMRConfig.masterInstanceType", | |
| "value" : null | |
| }, { | |
| "name" : "amazonEMRConfig.masterInstanceTypeCustom", | |
| "value" : null | |
| }, { | |
| "name" : "amazonEMRConfig.slaveInstanceType", | |
| "value" : null | |
| }, { | |
| "name" : "amazonEMRConfig.slaveInstanceTypeCustom", | |
| "value" : null | |
| } ], | |
| "uiInfo" : { | |
| "previewConfig" : { | |
| "previewSource" : "CONFIGURED_SOURCE", | |
| "batchSize" : 10, | |
| "timeout" : 30000, | |
| "writeToDestinations" : true, | |
| "executeLifecycleEvents" : false, | |
| "showHeader" : false, | |
| "showFieldType" : true, | |
| "rememberMe" : false | |
| } | |
| }, | |
| "fragments" : [ ], | |
| "stages" : [ { | |
| "instanceName" : "Directory_01", | |
| "library" : "streamsets-datacollector-basic-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_origin_spooldir_SpoolDirDSource", | |
| "stageVersion" : "10", | |
| "configuration" : [ { | |
| "name" : "conf.dataFormatConfig.compression", | |
| "value" : "NONE" | |
| }, { | |
| "name" : "conf.dataFormatConfig.filePatternInArchive", | |
| "value" : "*" | |
| }, { | |
| "name" : "conf.dataFormatConfig.charset", | |
| "value" : "UTF-8" | |
| }, { | |
| "name" : "conf.dataFormatConfig.removeCtrlChars", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.textMaxLineLen", | |
| "value" : 1024 | |
| }, { | |
| "name" : "conf.dataFormatConfig.useCustomDelimiter", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.customDelimiter", | |
| "value" : "\\r\\n" | |
| }, { | |
| "name" : "conf.dataFormatConfig.includeCustomDelimiterInTheText", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.jsonContent", | |
| "value" : "MULTIPLE_OBJECTS" | |
| }, { | |
| "name" : "conf.dataFormatConfig.jsonMaxObjectLen", | |
| "value" : 4096 | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvFileFormat", | |
| "value" : "CSV" | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvHeader", | |
| "value" : "NO_HEADER" | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvAllowExtraColumns", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvExtraColumnPrefix", | |
| "value" : "_extra_" | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvMaxObjectLen", | |
| "value" : 1024 | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvCustomDelimiter", | |
| "value" : "|" | |
| }, { | |
| "name" : "conf.dataFormatConfig.multiCharacterFieldDelimiter", | |
| "value" : "||" | |
| }, { | |
| "name" : "conf.dataFormatConfig.multiCharacterLineDelimiter", | |
| "value" : "${str:unescapeJava('\\\\n')}" | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvCustomEscape", | |
| "value" : "\\" | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvCustomQuote", | |
| "value" : "\"" | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvEnableComments", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvCommentMarker", | |
| "value" : "#" | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvIgnoreEmptyLines", | |
| "value" : true | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvRecordType", | |
| "value" : "LIST_MAP" | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvSkipStartLines", | |
| "value" : 0 | |
| }, { | |
| "name" : "conf.dataFormatConfig.parseNull", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.nullConstant", | |
| "value" : "\\\\N" | |
| }, { | |
| "name" : "conf.dataFormatConfig.xmlRecordElement", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.includeFieldXpathAttributes", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.xPathNamespaceContext", | |
| "value" : [ ] | |
| }, { | |
| "name" : "conf.dataFormatConfig.outputFieldAttributes", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.xmlMaxObjectLen", | |
| "value" : 4096 | |
| }, { | |
| "name" : "conf.dataFormatConfig.logMode", | |
| "value" : "COMBINED_LOG_FORMAT" | |
| }, { | |
| "name" : "conf.dataFormatConfig.logMaxObjectLen", | |
| "value" : 1024 | |
| }, { | |
| "name" : "conf.dataFormatConfig.retainOriginalLine", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.customLogFormat", | |
| "value" : "%h %l %u %t \"%r\" %>s %b" | |
| }, { | |
| "name" : "conf.dataFormatConfig.regex", | |
| "value" : "^(\\S+) (\\S+) (\\S+) \\[([\\w:/]+\\s[+\\-]\\d{4})\\] \"(\\S+) (\\S+) (\\S+)\" (\\d{3}) (\\d+)" | |
| }, { | |
| "name" : "conf.dataFormatConfig.fieldPathsToGroupName", | |
| "value" : [ { | |
| "fieldPath" : "/", | |
| "group" : 1 | |
| } ] | |
| }, { | |
| "name" : "conf.dataFormatConfig.grokPatternDefinition", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.grokPattern", | |
| "value" : "%{COMMONAPACHELOG}" | |
| }, { | |
| "name" : "conf.dataFormatConfig.onParseError", | |
| "value" : "ERROR" | |
| }, { | |
| "name" : "conf.dataFormatConfig.maxStackTraceLines", | |
| "value" : 50 | |
| }, { | |
| "name" : "conf.dataFormatConfig.enableLog4jCustomLogFormat", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.log4jCustomLogFormat", | |
| "value" : "%r [%t] %-5p %c %x - %m%n" | |
| }, { | |
| "name" : "conf.dataFormatConfig.avroSchema", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.schemaRegistryUrls", | |
| "value" : [ ] | |
| }, { | |
| "name" : "conf.dataFormatConfig.schemaLookupMode", | |
| "value" : "SUBJECT" | |
| }, { | |
| "name" : "conf.dataFormatConfig.subject", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.schemaId", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.avroSkipUnionIndex", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.protoDescriptorFile", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.messageType", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.isDelimited", | |
| "value" : true | |
| }, { | |
| "name" : "conf.dataFormatConfig.binaryMaxObjectLen", | |
| "value" : 1024 | |
| }, { | |
| "name" : "conf.dataFormatConfig.datagramMode", | |
| "value" : "SYSLOG" | |
| }, { | |
| "name" : "conf.dataFormatConfig.typesDbPath", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.convertTime", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.excludeInterval", | |
| "value" : true | |
| }, { | |
| "name" : "conf.dataFormatConfig.authFilePath", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.netflowOutputValuesMode", | |
| "value" : "RAW_AND_INTERPRETED" | |
| }, { | |
| "name" : "conf.dataFormatConfig.maxTemplateCacheSize", | |
| "value" : -1 | |
| }, { | |
| "name" : "conf.dataFormatConfig.templateCacheTimeoutMs", | |
| "value" : -1 | |
| }, { | |
| "name" : "conf.dataFormatConfig.netflowOutputValuesModeDatagram", | |
| "value" : "RAW_AND_INTERPRETED" | |
| }, { | |
| "name" : "conf.dataFormatConfig.maxTemplateCacheSizeDatagram", | |
| "value" : -1 | |
| }, { | |
| "name" : "conf.dataFormatConfig.templateCacheTimeoutMsDatagram", | |
| "value" : -1 | |
| }, { | |
| "name" : "conf.dataFormatConfig.wholeFileMaxObjectLen", | |
| "value" : 8192 | |
| }, { | |
| "name" : "conf.dataFormatConfig.rateLimit", | |
| "value" : "-1" | |
| }, { | |
| "name" : "conf.dataFormatConfig.excelHeader", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.excelSkipCellsWithNoHeader", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.excelReadAllSheets", | |
| "value" : true | |
| }, { | |
| "name" : "conf.dataFormatConfig.excelSheetNames", | |
| "value" : [ ] | |
| }, { | |
| "name" : "conf.dataFormat", | |
| "value" : "LOG" | |
| }, { | |
| "name" : "conf.spoolDir", | |
| "value" : "/resources" | |
| }, { | |
| "name" : "conf.numberOfThreads", | |
| "value" : 1 | |
| }, { | |
| "name" : "conf.pathMatcherMode", | |
| "value" : "GLOB" | |
| }, { | |
| "name" : "conf.filePattern", | |
| "value" : "*.log" | |
| }, { | |
| "name" : "conf.useLastModified", | |
| "value" : "LEXICOGRAPHICAL" | |
| }, { | |
| "name" : "conf.processSubdirectories", | |
| "value" : false | |
| }, { | |
| "name" : "conf.allowLateDirectory", | |
| "value" : false | |
| }, { | |
| "name" : "conf.overrunLimit", | |
| "value" : 128 | |
| }, { | |
| "name" : "conf.batchSize", | |
| "value" : 1000 | |
| }, { | |
| "name" : "conf.poolingTimeoutSecs", | |
| "value" : 60 | |
| }, { | |
| "name" : "conf.maxSpoolFiles", | |
| "value" : 1000 | |
| }, { | |
| "name" : "conf.spoolingPeriod", | |
| "value" : 5 | |
| }, { | |
| "name" : "conf.initialFileToProcess", | |
| "value" : null | |
| }, { | |
| "name" : "conf.errorArchiveDir", | |
| "value" : null | |
| }, { | |
| "name" : "conf.postProcessing", | |
| "value" : "NONE" | |
| }, { | |
| "name" : "conf.archiveDir", | |
| "value" : null | |
| }, { | |
| "name" : "conf.retentionTimeMins", | |
| "value" : 0 | |
| }, { | |
| "name" : "stageOnRecordError", | |
| "value" : "TO_ERROR" | |
| } ], | |
| "uiInfo" : { | |
| "yPos" : 50, | |
| "stageType" : "SOURCE", | |
| "rawSource" : { | |
| "configuration" : [ { | |
| "name" : "fileName" | |
| } ] | |
| }, | |
| "icon" : "directory.png", | |
| "description" : "", | |
| "label" : "Logs @ Origin", | |
| "xPos" : 60 | |
| }, | |
| "inputLanes" : [ ], | |
| "outputLanes" : [ "Directory_01OutputLane15753283024460" ], | |
| "eventLanes" : [ ], | |
| "services" : [ ] | |
| }, { | |
| "instanceName" : "ExpressionEvaluator_01", | |
| "library" : "streamsets-datacollector-basic-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_processor_expression_ExpressionDProcessor", | |
| "stageVersion" : "2", | |
| "configuration" : [ { | |
| "name" : "expressionProcessorConfigs", | |
| "value" : [ { | |
| "fieldToSet" : "/geo", | |
| "expression" : "${emptyMap()}" | |
| } ] | |
| }, { | |
| "name" : "headerAttributeConfigs", | |
| "value" : [ { } ] | |
| }, { | |
| "name" : "fieldAttributeConfigs", | |
| "value" : [ { | |
| "fieldToSet" : "/" | |
| } ] | |
| }, { | |
| "name" : "stageOnRecordError", | |
| "value" : "TO_ERROR" | |
| }, { | |
| "name" : "stageRequiredFields", | |
| "value" : [ ] | |
| }, { | |
| "name" : "stageRecordPreconditions", | |
| "value" : [ ] | |
| } ], | |
| "uiInfo" : { | |
| "yPos" : 50, | |
| "stageType" : "PROCESSOR", | |
| "icon" : "expression.png", | |
| "description" : "", | |
| "label" : "GeoIP", | |
| "xPos" : 280 | |
| }, | |
| "inputLanes" : [ "Directory_01OutputLane15753283024460" ], | |
| "outputLanes" : [ "ExpressionEvaluator_01OutputLane15753285817100" ], | |
| "eventLanes" : [ ], | |
| "services" : [ ] | |
| }, { | |
| "instanceName" : "FieldTypeConverter_01", | |
| "library" : "streamsets-datacollector-basic-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_processor_fieldtypeconverter_FieldTypeConverterDProcessor", | |
| "stageVersion" : "2", | |
| "configuration" : [ { | |
| "name" : "convertBy", | |
| "value" : "BY_FIELD" | |
| }, { | |
| "name" : "fieldTypeConverterConfigs", | |
| "value" : [ { | |
| "fields" : [ "/bytes" ], | |
| "targetType" : "LONG", | |
| "treatInputFieldAsDate" : false, | |
| "dataLocale" : "en,US", | |
| "scale" : -1, | |
| "decimalScaleRoundingStrategy" : "ROUND_UNNECESSARY", | |
| "dateFormat" : "YYYY_MM_DD", | |
| "zonedDateTimeFormat" : "ISO_ZONED_DATE_TIME", | |
| "encoding" : "UTF-8" | |
| }, { | |
| "fields" : [ "/response" ], | |
| "targetType" : "INTEGER", | |
| "treatInputFieldAsDate" : false, | |
| "dataLocale" : "en,US", | |
| "scale" : -1, | |
| "decimalScaleRoundingStrategy" : "ROUND_UNNECESSARY", | |
| "dateFormat" : "YYYY_MM_DD", | |
| "zonedDateTimeFormat" : "ISO_ZONED_DATE_TIME", | |
| "encoding" : "UTF-8" | |
| }, { | |
| "fields" : [ "/timestamp" ], | |
| "targetType" : "DATETIME", | |
| "treatInputFieldAsDate" : false, | |
| "dataLocale" : "en,US", | |
| "scale" : -1, | |
| "decimalScaleRoundingStrategy" : "ROUND_UNNECESSARY", | |
| "dateFormat" : "OTHER", | |
| "zonedDateTimeFormat" : "ISO_ZONED_DATE_TIME", | |
| "encoding" : "UTF-8", | |
| "otherDateFormat" : "dd/MMM/y:H:m:s Z" | |
| } ] | |
| }, { | |
| "name" : "wholeTypeConverterConfigs", | |
| "value" : [ { | |
| "sourceType" : "INTEGER", | |
| "targetType" : "INTEGER", | |
| "treatInputFieldAsDate" : false, | |
| "dataLocale" : "en,US", | |
| "scale" : -1, | |
| "decimalScaleRoundingStrategy" : "ROUND_UNNECESSARY", | |
| "dateFormat" : "YYYY_MM_DD", | |
| "zonedDateTimeFormat" : "ISO_ZONED_DATE_TIME", | |
| "encoding" : "UTF-8" | |
| } ] | |
| }, { | |
| "name" : "stageOnRecordError", | |
| "value" : "TO_ERROR" | |
| }, { | |
| "name" : "stageRequiredFields", | |
| "value" : [ ] | |
| }, { | |
| "name" : "stageRecordPreconditions", | |
| "value" : [ ] | |
| } ], | |
| "uiInfo" : { | |
| "yPos" : 50, | |
| "stageType" : "PROCESSOR", | |
| "icon" : "converter.png", | |
| "description" : "", | |
| "label" : "Field Type Converter 1", | |
| "xPos" : 500 | |
| }, | |
| "inputLanes" : [ "ExpressionEvaluator_01OutputLane15753285817100" ], | |
| "outputLanes" : [ "FieldTypeConverter_01OutputLane15753286919090" ], | |
| "eventLanes" : [ ], | |
| "services" : [ ] | |
| }, { | |
| "instanceName" : "GeoIP_01", | |
| "library" : "streamsets-datacollector-basic-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_processor_geolocation_GeolocationDProcessor", | |
| "stageVersion" : "4", | |
| "configuration" : [ { | |
| "name" : "dbConfigs", | |
| "value" : [ { | |
| "geoIP2DBFile" : "GeoLite2-City.mmdb", | |
| "geoIP2DBType" : "CITY" | |
| } ] | |
| }, { | |
| "name" : "fieldTypeConverterConfigs", | |
| "value" : [ { | |
| "targetType" : "CITY_NAME", | |
| "inputFieldName" : "/clientip", | |
| "outputFieldName" : "/city" | |
| }, { | |
| "targetType" : "LATITUDE", | |
| "inputFieldName" : "/clientip", | |
| "outputFieldName" : "/geo/lat" | |
| }, { | |
| "targetType" : "LONGITUDE", | |
| "inputFieldName" : "/clientip", | |
| "outputFieldName" : "/geo/lon" | |
| } ] | |
| }, { | |
| "name" : "missingAddressAction", | |
| "value" : "TO_ERROR" | |
| }, { | |
| "name" : "stageOnRecordError", | |
| "value" : "TO_ERROR" | |
| }, { | |
| "name" : "stageRequiredFields", | |
| "value" : [ ] | |
| }, { | |
| "name" : "stageRecordPreconditions", | |
| "value" : [ ] | |
| } ], | |
| "uiInfo" : { | |
| "yPos" : 50, | |
| "stageType" : "PROCESSOR", | |
| "icon" : "globe.png", | |
| "description" : "", | |
| "label" : "Geo IP 1", | |
| "xPos" : 720 | |
| }, | |
| "inputLanes" : [ "FieldTypeConverter_01OutputLane15753286919090" ], | |
| "outputLanes" : [ "GeoIP_01OutputLane15753288954810" ], | |
| "eventLanes" : [ ], | |
| "services" : [ ] | |
| }, { | |
| "instanceName" : "Elasticsearch_01", | |
| "library" : "streamsets-datacollector-elasticsearch_5-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_destination_elasticsearch_ElasticSearchDTarget", | |
| "stageVersion" : "10", | |
| "configuration" : [ { | |
| "name" : "elasticSearchConfig.timeDriver", | |
| "value" : "${time:now()}" | |
| }, { | |
| "name" : "elasticSearchConfig.timeZoneID", | |
| "value" : "UTC" | |
| }, { | |
| "name" : "elasticSearchConfig.indexTemplate", | |
| "value" : "logs" | |
| }, { | |
| "name" : "elasticSearchConfig.typeTemplate", | |
| "value" : "_doc" | |
| }, { | |
| "name" : "elasticSearchConfig.docIdTemplate", | |
| "value" : null | |
| }, { | |
| "name" : "elasticSearchConfig.parentIdTemplate", | |
| "value" : null | |
| }, { | |
| "name" : "elasticSearchConfig.routingTemplate", | |
| "value" : null | |
| }, { | |
| "name" : "elasticSearchConfig.charset", | |
| "value" : "UTF-8" | |
| }, { | |
| "name" : "elasticSearchConfig.defaultOperation", | |
| "value" : "INDEX" | |
| }, { | |
| "name" : "elasticSearchConfig.unsupportedAction", | |
| "value" : "DISCARD" | |
| }, { | |
| "name" : "elasticSearchConfig.rawAdditionalProperties", | |
| "value" : "{\n}" | |
| }, { | |
| "name" : "elasticSearchConfig.securityConfig.securityMode", | |
| "value" : "BASIC" | |
| }, { | |
| "name" : "elasticSearchConfig.securityConfig.awsRegion", | |
| "value" : null | |
| }, { | |
| "name" : "elasticSearchConfig.securityConfig.endpoint", | |
| "value" : null | |
| }, { | |
| "name" : "elasticSearchConfig.securityConfig.awsAccessKeyId", | |
| "value" : "" | |
| }, { | |
| "name" : "elasticSearchConfig.securityConfig.awsSecretAccessKey", | |
| "value" : "" | |
| }, { | |
| "name" : "elasticSearchConfig.securityConfig.securityUser", | |
| "value" : "" | |
| }, { | |
| "name" : "elasticSearchConfig.securityConfig.sslTrustStorePath", | |
| "value" : null | |
| }, { | |
| "name" : "elasticSearchConfig.securityConfig.sslTrustStorePassword", | |
| "value" : "" | |
| }, { | |
| "name" : "elasticSearchConfig.httpUris", | |
| "value" : [ "https://es-sstutorial-business-demo.aivencloud.com:18787" ] | |
| }, { | |
| "name" : "elasticSearchConfig.params", | |
| "value" : [ ] | |
| }, { | |
| "name" : "elasticSearchConfig.clientSniff", | |
| "value" : false | |
| }, { | |
| "name" : "elasticSearchConfig.useSecurity", | |
| "value" : true | |
| }, { | |
| "name" : "stageOnRecordError", | |
| "value" : "TO_ERROR" | |
| }, { | |
| "name" : "stageRequiredFields", | |
| "value" : [ ] | |
| }, { | |
| "name" : "stageRecordPreconditions", | |
| "value" : [ ] | |
| } ], | |
| "uiInfo" : { | |
| "yPos" : 50, | |
| "stageType" : "TARGET", | |
| "icon" : "elasticsearch.png", | |
| "description" : "", | |
| "label" : "Aiven Elasticsearch", | |
| "xPos" : 940 | |
| }, | |
| "inputLanes" : [ "GeoIP_01OutputLane15753288954810" ], | |
| "outputLanes" : [ ], | |
| "eventLanes" : [ ], | |
| "services" : [ ] | |
| }, { | |
| "instanceName" : "ExpressionEvaluator_02", | |
| "library" : "streamsets-datacollector-basic-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_processor_expression_ExpressionDProcessor", | |
| "stageVersion" : "2", | |
| "configuration" : [ { | |
| "name" : "expressionProcessorConfigs", | |
| "value" : [ { | |
| "fieldToSet" : "/uuid", | |
| "expression" : "${uuid:uuid()}" | |
| } ] | |
| }, { | |
| "name" : "headerAttributeConfigs", | |
| "value" : [ { } ] | |
| }, { | |
| "name" : "fieldAttributeConfigs", | |
| "value" : [ { | |
| "fieldToSet" : "/" | |
| } ] | |
| }, { | |
| "name" : "stageOnRecordError", | |
| "value" : "TO_ERROR" | |
| }, { | |
| "name" : "stageRequiredFields", | |
| "value" : [ ] | |
| }, { | |
| "name" : "stageRecordPreconditions", | |
| "value" : [ ] | |
| } ], | |
| "uiInfo" : { | |
| "yPos" : 200, | |
| "stageType" : "PROCESSOR", | |
| "icon" : "expression.png", | |
| "description" : "", | |
| "label" : "Expression Evaluator 1", | |
| "xPos" : 940 | |
| }, | |
| "inputLanes" : [ "GeoIP_01OutputLane15753288954810" ], | |
| "outputLanes" : [ "ExpressionEvaluator_02OutputLane15755044832890" ], | |
| "eventLanes" : [ ], | |
| "services" : [ ] | |
| }, { | |
| "instanceName" : "FieldRenamer_01", | |
| "library" : "streamsets-datacollector-basic-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_processor_fieldrenamer_FieldRenamerDProcessor", | |
| "stageVersion" : "2", | |
| "configuration" : [ { | |
| "name" : "renameMapping", | |
| "value" : [ { | |
| "fromFieldExpression" : "/city", | |
| "toFieldExpression" : "/city_name" | |
| } ] | |
| }, { | |
| "name" : "errorHandler.nonExistingFromFieldHandling", | |
| "value" : "TO_ERROR" | |
| }, { | |
| "name" : "errorHandler.existingToFieldHandling", | |
| "value" : "TO_ERROR" | |
| }, { | |
| "name" : "errorHandler.multipleFromFieldsMatching", | |
| "value" : "TO_ERROR" | |
| }, { | |
| "name" : "stageOnRecordError", | |
| "value" : "TO_ERROR" | |
| }, { | |
| "name" : "stageRequiredFields", | |
| "value" : [ ] | |
| }, { | |
| "name" : "stageRecordPreconditions", | |
| "value" : [ ] | |
| } ], | |
| "uiInfo" : { | |
| "yPos" : 200, | |
| "stageType" : "PROCESSOR", | |
| "icon" : "edit.png", | |
| "description" : "", | |
| "label" : "Field Renamer 1", | |
| "xPos" : 1160 | |
| }, | |
| "inputLanes" : [ "ExpressionEvaluator_02OutputLane15755044832890" ], | |
| "outputLanes" : [ "FieldRenamer_01OutputLane15755039867100" ], | |
| "eventLanes" : [ ], | |
| "services" : [ ] | |
| }, { | |
| "instanceName" : "Cassandra_01", | |
| "library" : "streamsets-datacollector-cassandra_3-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_destination_cassandra_CassandraDTarget", | |
| "stageVersion" : "5", | |
| "configuration" : [ { | |
| "name" : "conf.contactPoints", | |
| "value" : [ "cassandra-sstutorial-business-demo.aivencloud.com" ] | |
| }, { | |
| "name" : "conf.port", | |
| "value" : 18787 | |
| }, { | |
| "name" : "conf.authProviderOption", | |
| "value" : "PLAINTEXT" | |
| }, { | |
| "name" : "conf.protocolVersion", | |
| "value" : "V4" | |
| }, { | |
| "name" : "conf.compression", | |
| "value" : "LZ4" | |
| }, { | |
| "name" : "conf.batchType", | |
| "value" : "LOGGED" | |
| }, { | |
| "name" : "conf.maxBatchSize", | |
| "value" : 50 | |
| }, { | |
| "name" : "conf.qualifiedTableName", | |
| "value" : "location.city" | |
| }, { | |
| "name" : "conf.columnNames", | |
| "value" : [ { | |
| "field" : "/city_name", | |
| "columnName" : "city_name" | |
| }, { | |
| "field" : "/referrer", | |
| "columnName" : "referrer" | |
| }, { | |
| "field" : "/uuid", | |
| "columnName" : "uuid" | |
| } ] | |
| }, { | |
| "name" : "conf.connectionTimeout", | |
| "value" : 5000 | |
| }, { | |
| "name" : "conf.readTimeout", | |
| "value" : 5000 | |
| }, { | |
| "name" : "conf.consistencyLevel", | |
| "value" : "LOCAL_ONE" | |
| }, { | |
| "name" : "conf.logSlowQueries", | |
| "value" : false | |
| }, { | |
| "name" : "conf.slowQueryThreshold", | |
| "value" : 5000 | |
| }, { | |
| "name" : "conf.disableBatchInsert", | |
| "value" : false | |
| }, { | |
| "name" : "conf.requestTimeout", | |
| "value" : 5000 | |
| }, { | |
| "name" : "conf.username", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.password", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.tlsConfig.tlsEnabled", | |
| "value" : true | |
| }, { | |
| "name" : "conf.tlsConfig.keyStoreFilePath", | |
| "value" : null | |
| }, { | |
| "name" : "conf.tlsConfig.keyStoreType", | |
| "value" : "JKS" | |
| }, { | |
| "name" : "conf.tlsConfig.keyStorePassword", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.tlsConfig.keyStoreAlgorithm", | |
| "value" : "SunX509" | |
| }, { | |
| "name" : "conf.tlsConfig.trustStoreFilePath", | |
| "value" : "truststore.jks" | |
| }, { | |
| "name" : "conf.tlsConfig.trustStoreType", | |
| "value" : "JKS" | |
| }, { | |
| "name" : "conf.tlsConfig.trustStorePassword", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.tlsConfig.trustStoreAlgorithm", | |
| "value" : "SunX509" | |
| }, { | |
| "name" : "conf.tlsConfig.useDefaultProtocols", | |
| "value" : true | |
| }, { | |
| "name" : "conf.tlsConfig.protocols", | |
| "value" : [ ] | |
| }, { | |
| "name" : "conf.tlsConfig.useDefaultCiperSuites", | |
| "value" : true | |
| }, { | |
| "name" : "conf.tlsConfig.cipherSuites", | |
| "value" : [ ] | |
| }, { | |
| "name" : "stageOnRecordError", | |
| "value" : "TO_ERROR" | |
| }, { | |
| "name" : "stageRequiredFields", | |
| "value" : [ ] | |
| }, { | |
| "name" : "stageRecordPreconditions", | |
| "value" : [ ] | |
| } ], | |
| "uiInfo" : { | |
| "yPos" : 200, | |
| "stageType" : "TARGET", | |
| "icon" : "cassandra.png", | |
| "description" : "", | |
| "label" : "Aiven for Cassandra", | |
| "xPos" : 1380 | |
| }, | |
| "inputLanes" : [ "FieldRenamer_01OutputLane15755039867100" ], | |
| "outputLanes" : [ ], | |
| "eventLanes" : [ ], | |
| "services" : [ ] | |
| } ], | |
| "errorStage" : { | |
| "instanceName" : "Discard_ErrorStage", | |
| "library" : "streamsets-datacollector-basic-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget", | |
| "stageVersion" : "1", | |
| "configuration" : [ ], | |
| "uiInfo" : { | |
| "yPos" : 50, | |
| "stageType" : "TARGET", | |
| "icon" : "", | |
| "description" : "", | |
| "label" : "Error Records - Discard", | |
| "xPos" : 1160 | |
| }, | |
| "inputLanes" : [ ], | |
| "outputLanes" : [ ], | |
| "eventLanes" : [ ], | |
| "services" : [ ] | |
| }, | |
| "info" : { | |
| "pipelineId" : "LogsfromSSDCtoElasticsearch42654be3-3013-4bc0-b981-75e066aec41c", | |
| "title" : "Logs_from_SSDC_to_Elasticsearch_and_Cassandra", | |
| "description" : "Pipeline from tutorial 1", | |
| "created" : 1575328260829, | |
| "lastModified" : 1575515922225, | |
| "creator" : "admin", | |
| "lastModifier" : "admin", | |
| "lastRev" : "0", | |
| "uuid" : "cbbbfff6-4a1a-4548-b623-3f452c816502", | |
| "valid" : true, | |
| "metadata" : { | |
| "labels" : [ ] | |
| }, | |
| "name" : "LogsfromSSDCtoElasticsearch42654be3-3013-4bc0-b981-75e066aec41c", | |
| "sdcVersion" : "3.11.0", | |
| "sdcId" : "f8973afe-1557-11ea-80e9-537b1b32fe22" | |
| }, | |
| "metadata" : { | |
| "labels" : [ ] | |
| }, | |
| "statsAggregatorStage" : { | |
| "instanceName" : "statsAggregatorStageInstance", | |
| "library" : "streamsets-datacollector-basic-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_destination_devnull_StatsDpmDirectlyDTarget", | |
| "stageVersion" : "1", | |
| "configuration" : [ ], | |
| "uiInfo" : { | |
| "stageType" : "TARGET", | |
| "label" : "Stats Aggregator -Write Directly to Control Hub - statistics are not aggregated across Data Collectors" | |
| }, | |
| "inputLanes" : [ ], | |
| "outputLanes" : [ ], | |
| "eventLanes" : [ ], | |
| "services" : [ ] | |
| }, | |
| "startEventStages" : [ { | |
| "instanceName" : "Discard_StartEventStage", | |
| "library" : "streamsets-datacollector-basic-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget", | |
| "stageVersion" : "1", | |
| "configuration" : [ ], | |
| "uiInfo" : { | |
| "yPos" : 50, | |
| "stageType" : "TARGET", | |
| "icon" : "", | |
| "description" : "", | |
| "label" : "Start Event - Discard", | |
| "xPos" : 280 | |
| }, | |
| "inputLanes" : [ ], | |
| "outputLanes" : [ ], | |
| "eventLanes" : [ ], | |
| "services" : [ ] | |
| } ], | |
| "stopEventStages" : [ { | |
| "instanceName" : "Discard_StopEventStage", | |
| "library" : "streamsets-datacollector-basic-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget", | |
| "stageVersion" : "1", | |
| "configuration" : [ ], | |
| "uiInfo" : { | |
| "yPos" : 50, | |
| "stageType" : "TARGET", | |
| "icon" : "", | |
| "description" : "", | |
| "label" : "Stop Event - Discard", | |
| "xPos" : 280 | |
| }, | |
| "inputLanes" : [ ], | |
| "outputLanes" : [ ], | |
| "eventLanes" : [ ], | |
| "services" : [ ] | |
| } ], | |
| "testOriginStage" : { | |
| "instanceName" : "com_streamsets_pipeline_stage_devtest_rawdata_RawDataDSource_TestOriginStage", | |
| "library" : "streamsets-datacollector-dev-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_devtest_rawdata_RawDataDSource", | |
| "stageVersion" : "3", | |
| "configuration" : [ { | |
| "name" : "rawData", | |
| "value" : "{\n \"f1\": \"abc\",\n \"f2\": \"xyz\",\n \"f3\": \"lmn\"\n}" | |
| }, { | |
| "name" : "stopAfterFirstBatch", | |
| "value" : false | |
| }, { | |
| "name" : "eventData", | |
| "value" : null | |
| }, { | |
| "name" : "stageOnRecordError", | |
| "value" : "TO_ERROR" | |
| } ], | |
| "uiInfo" : { | |
| "stageType" : "SOURCE", | |
| "label" : "Test Origin - Dev Raw Data Source" | |
| }, | |
| "inputLanes" : [ ], | |
| "outputLanes" : [ "com_streamsets_pipeline_stage_devtest_rawdata_RawDataDSource_TestOriginStageOutputLane1" ], | |
| "eventLanes" : [ ], | |
| "services" : [ { | |
| "service" : "com.streamsets.pipeline.api.service.dataformats.DataFormatParserService", | |
| "serviceVersion" : 1, | |
| "configuration" : [ { | |
| "name" : "displayFormats", | |
| "value" : "DELIMITED,JSON,LOG,SDC_JSON,TEXT,XML" | |
| }, { | |
| "name" : "dataFormat", | |
| "value" : "JSON" | |
| }, { | |
| "name" : "dataFormatConfig.compression", | |
| "value" : "NONE" | |
| }, { | |
| "name" : "dataFormatConfig.filePatternInArchive", | |
| "value" : "*" | |
| }, { | |
| "name" : "dataFormatConfig.charset", | |
| "value" : "UTF-8" | |
| }, { | |
| "name" : "dataFormatConfig.removeCtrlChars", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.textMaxLineLen", | |
| "value" : 1024 | |
| }, { | |
| "name" : "dataFormatConfig.useCustomDelimiter", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.customDelimiter", | |
| "value" : "\\r\\n" | |
| }, { | |
| "name" : "dataFormatConfig.includeCustomDelimiterInTheText", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.jsonContent", | |
| "value" : "MULTIPLE_OBJECTS" | |
| }, { | |
| "name" : "dataFormatConfig.jsonMaxObjectLen", | |
| "value" : 4096 | |
| }, { | |
| "name" : "dataFormatConfig.csvFileFormat", | |
| "value" : "CSV" | |
| }, { | |
| "name" : "dataFormatConfig.csvHeader", | |
| "value" : "NO_HEADER" | |
| }, { | |
| "name" : "dataFormatConfig.csvAllowExtraColumns", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.csvExtraColumnPrefix", | |
| "value" : "_extra_" | |
| }, { | |
| "name" : "dataFormatConfig.csvMaxObjectLen", | |
| "value" : 1024 | |
| }, { | |
| "name" : "dataFormatConfig.csvCustomDelimiter", | |
| "value" : "|" | |
| }, { | |
| "name" : "dataFormatConfig.multiCharacterFieldDelimiter", | |
| "value" : "||" | |
| }, { | |
| "name" : "dataFormatConfig.multiCharacterLineDelimiter", | |
| "value" : "${str:unescapeJava('\\\\n')}" | |
| }, { | |
| "name" : "dataFormatConfig.csvCustomEscape", | |
| "value" : "\\" | |
| }, { | |
| "name" : "dataFormatConfig.csvCustomQuote", | |
| "value" : "\"" | |
| }, { | |
| "name" : "dataFormatConfig.csvEnableComments", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.csvCommentMarker", | |
| "value" : "#" | |
| }, { | |
| "name" : "dataFormatConfig.csvIgnoreEmptyLines", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.csvRecordType", | |
| "value" : "LIST_MAP" | |
| }, { | |
| "name" : "dataFormatConfig.csvSkipStartLines", | |
| "value" : 0 | |
| }, { | |
| "name" : "dataFormatConfig.parseNull", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.nullConstant", | |
| "value" : "\\\\N" | |
| }, { | |
| "name" : "dataFormatConfig.xmlRecordElement", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.includeFieldXpathAttributes", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.xPathNamespaceContext", | |
| "value" : [ ] | |
| }, { | |
| "name" : "dataFormatConfig.outputFieldAttributes", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.xmlMaxObjectLen", | |
| "value" : 4096 | |
| }, { | |
| "name" : "dataFormatConfig.logMode", | |
| "value" : "COMMON_LOG_FORMAT" | |
| }, { | |
| "name" : "dataFormatConfig.logMaxObjectLen", | |
| "value" : 1024 | |
| }, { | |
| "name" : "dataFormatConfig.retainOriginalLine", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.customLogFormat", | |
| "value" : "%h %l %u %t \"%r\" %>s %b" | |
| }, { | |
| "name" : "dataFormatConfig.regex", | |
| "value" : "^(\\S+) (\\S+) (\\S+) \\[([\\w:/]+\\s[+\\-]\\d{4})\\] \"(\\S+) (\\S+) (\\S+)\" (\\d{3}) (\\d+)" | |
| }, { | |
| "name" : "dataFormatConfig.fieldPathsToGroupName", | |
| "value" : [ { | |
| "fieldPath" : "/", | |
| "group" : 1 | |
| } ] | |
| }, { | |
| "name" : "dataFormatConfig.grokPatternDefinition", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.grokPattern", | |
| "value" : "%{COMMONAPACHELOG}" | |
| }, { | |
| "name" : "dataFormatConfig.onParseError", | |
| "value" : "ERROR" | |
| }, { | |
| "name" : "dataFormatConfig.maxStackTraceLines", | |
| "value" : 50 | |
| }, { | |
| "name" : "dataFormatConfig.enableLog4jCustomLogFormat", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.log4jCustomLogFormat", | |
| "value" : "%r [%t] %-5p %c %x - %m%n" | |
| }, { | |
| "name" : "dataFormatConfig.avroSchemaSource", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.avroSchema", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.schemaRegistryUrls", | |
| "value" : [ ] | |
| }, { | |
| "name" : "dataFormatConfig.schemaLookupMode", | |
| "value" : "SUBJECT" | |
| }, { | |
| "name" : "dataFormatConfig.subject", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.schemaId", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.avroSkipUnionIndex", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.protoDescriptorFile", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.messageType", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.isDelimited", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.binaryMaxObjectLen", | |
| "value" : 1024 | |
| }, { | |
| "name" : "dataFormatConfig.datagramMode", | |
| "value" : "SYSLOG" | |
| }, { | |
| "name" : "dataFormatConfig.typesDbPath", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.convertTime", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.excludeInterval", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.authFilePath", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.netflowOutputValuesMode", | |
| "value" : "RAW_AND_INTERPRETED" | |
| }, { | |
| "name" : "dataFormatConfig.maxTemplateCacheSize", | |
| "value" : -1 | |
| }, { | |
| "name" : "dataFormatConfig.templateCacheTimeoutMs", | |
| "value" : -1 | |
| }, { | |
| "name" : "dataFormatConfig.netflowOutputValuesModeDatagram", | |
| "value" : "RAW_AND_INTERPRETED" | |
| }, { | |
| "name" : "dataFormatConfig.maxTemplateCacheSizeDatagram", | |
| "value" : -1 | |
| }, { | |
| "name" : "dataFormatConfig.templateCacheTimeoutMsDatagram", | |
| "value" : -1 | |
| }, { | |
| "name" : "dataFormatConfig.wholeFileMaxObjectLen", | |
| "value" : 8192 | |
| }, { | |
| "name" : "dataFormatConfig.rateLimit", | |
| "value" : "-1" | |
| }, { | |
| "name" : "dataFormatConfig.verifyChecksum", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.excelHeader", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.excelSkipCellsWithNoHeader", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.excelReadAllSheets", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.excelSheetNames", | |
| "value" : [ ] | |
| } ] | |
| } ] | |
| }, | |
| "valid" : true, | |
| "issues" : { | |
| "pipelineIssues" : [ ], | |
| "stageIssues" : { }, | |
| "issueCount" : 0 | |
| }, | |
| "previewable" : true | |
| }, | |
| "pipelineRules" : { | |
| "schemaVersion" : 3, | |
| "version" : 2, | |
| "metricsRuleDefinitions" : [ { | |
| "id" : "badRecordsAlertID", | |
| "alertText" : "High incidence of Error Records", | |
| "metricId" : "pipeline.batchErrorRecords.counter", | |
| "metricType" : "COUNTER", | |
| "metricElement" : "COUNTER_COUNT", | |
| "condition" : "${value() > 100}", | |
| "sendEmail" : false, | |
| "enabled" : false, | |
| "timestamp" : 1575328260961, | |
| "valid" : true | |
| }, { | |
| "id" : "stageErrorAlertID", | |
| "alertText" : "High incidence of Stage Errors", | |
| "metricId" : "pipeline.batchErrorMessages.counter", | |
| "metricType" : "COUNTER", | |
| "metricElement" : "COUNTER_COUNT", | |
| "condition" : "${value() > 100}", | |
| "sendEmail" : false, | |
| "enabled" : false, | |
| "timestamp" : 1575328260961, | |
| "valid" : true | |
| }, { | |
| "id" : "idleGaugeID", | |
| "alertText" : "Pipeline is Idle", | |
| "metricId" : "RuntimeStatsGauge.gauge", | |
| "metricType" : "GAUGE", | |
| "metricElement" : "TIME_OF_LAST_RECEIVED_RECORD", | |
| "condition" : "${time:now() - value() > 120000}", | |
| "sendEmail" : false, | |
| "enabled" : false, | |
| "timestamp" : 1575328260961, | |
| "valid" : true | |
| }, { | |
| "id" : "batchTimeAlertID", | |
| "alertText" : "Batch taking more time to process", | |
| "metricId" : "RuntimeStatsGauge.gauge", | |
| "metricType" : "GAUGE", | |
| "metricElement" : "CURRENT_BATCH_AGE", | |
| "condition" : "${value() > 200}", | |
| "sendEmail" : false, | |
| "enabled" : false, | |
| "timestamp" : 1575328260961, | |
| "valid" : true | |
| } ], | |
| "dataRuleDefinitions" : [ ], | |
| "driftRuleDefinitions" : [ ], | |
| "uuid" : "d64e5e78-db92-4717-8c45-5d6cc54d91a7", | |
| "configuration" : [ { | |
| "name" : "emailIDs", | |
| "value" : [ ] | |
| }, { | |
| "name" : "webhookConfigs", | |
| "value" : [ ] | |
| } ], | |
| "configIssues" : [ ], | |
| "ruleIssues" : [ ] | |
| }, | |
| "libraryDefinitions" : null | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment