Created
August 20, 2019 18:09
-
-
Save metadaddy/50d62481054004907123040493318d43 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "pipelineConfig" : { | |
| "schemaVersion" : 6, | |
| "version" : 12, | |
| "pipelineId" : "personalbox4f3590a2-1b4e-40ac-beae-600e976ef496", | |
| "title" : "personal_box", | |
| "description" : "", | |
| "uuid" : "043a4e85-7989-4761-90de-e68401257e9e", | |
| "configuration" : [ { | |
| "name" : "executionMode", | |
| "value" : "STANDALONE" | |
| }, { | |
| "name" : "edgeHttpUrl", | |
| "value" : "http://localhost:18633" | |
| }, { | |
| "name" : "deliveryGuarantee", | |
| "value" : "AT_LEAST_ONCE" | |
| }, { | |
| "name" : "testOriginStage", | |
| "value" : "streamsets-datacollector-dev-lib::com_streamsets_pipeline_stage_devtest_rawdata_RawDataDSource::3" | |
| }, { | |
| "name" : "startEventStage", | |
| "value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1" | |
| }, { | |
| "name" : "stopEventStage", | |
| "value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1" | |
| }, { | |
| "name" : "shouldRetry", | |
| "value" : true | |
| }, { | |
| "name" : "retryAttempts", | |
| "value" : -1 | |
| }, { | |
| "name" : "notifyOnStates", | |
| "value" : [ "RUN_ERROR", "STOPPED", "FINISHED" ] | |
| }, { | |
| "name" : "emailIDs", | |
| "value" : [ ] | |
| }, { | |
| "name" : "constants", | |
| "value" : [ ] | |
| }, { | |
| "name" : "badRecordsHandling", | |
| "value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1" | |
| }, { | |
| "name" : "errorRecordPolicy", | |
| "value" : "ORIGINAL_RECORD" | |
| }, { | |
| "name" : "statsAggregatorStage", | |
| "value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_StatsDpmDirectlyDTarget::1" | |
| }, { | |
| "name" : "workerCount", | |
| "value" : 0 | |
| }, { | |
| "name" : "clusterSlaveMemory", | |
| "value" : 2048 | |
| }, { | |
| "name" : "clusterSlaveJavaOpts", | |
| "value" : "-XX:+UseConcMarkSweepGC -XX:+UseParNewGC -Dlog4j.debug" | |
| }, { | |
| "name" : "clusterLauncherEnv", | |
| "value" : [ ] | |
| }, { | |
| "name" : "mesosDispatcherURL", | |
| "value" : null | |
| }, { | |
| "name" : "logLevel", | |
| "value" : "INFO" | |
| }, { | |
| "name" : "hdfsS3ConfDir", | |
| "value" : null | |
| }, { | |
| "name" : "rateLimit", | |
| "value" : 0 | |
| }, { | |
| "name" : "maxRunners", | |
| "value" : 0 | |
| }, { | |
| "name" : "shouldCreateFailureSnapshot", | |
| "value" : true | |
| }, { | |
| "name" : "runnerIdleTIme", | |
| "value" : 60 | |
| }, { | |
| "name" : "webhookConfigs", | |
| "value" : [ ] | |
| }, { | |
| "name" : "sparkConfigs", | |
| "value" : [ ] | |
| }, { | |
| "name" : "amazonEMRConfig.userRegion", | |
| "value" : null | |
| }, { | |
| "name" : "amazonEMRConfig.userRegionCustom", | |
| "value" : null | |
| }, { | |
| "name" : "amazonEMRConfig.accessKey", | |
| "value" : "" | |
| }, { | |
| "name" : "amazonEMRConfig.secretKey", | |
| "value" : "" | |
| }, { | |
| "name" : "amazonEMRConfig.s3StagingUri", | |
| "value" : null | |
| }, { | |
| "name" : "amazonEMRConfig.provisionNewCluster", | |
| "value" : false | |
| }, { | |
| "name" : "amazonEMRConfig.clusterId", | |
| "value" : null | |
| }, { | |
| "name" : "amazonEMRConfig.clusterPrefix", | |
| "value" : null | |
| }, { | |
| "name" : "amazonEMRConfig.terminateCluster", | |
| "value" : false | |
| }, { | |
| "name" : "amazonEMRConfig.loggingEnabled", | |
| "value" : true | |
| }, { | |
| "name" : "amazonEMRConfig.s3LogUri", | |
| "value" : null | |
| }, { | |
| "name" : "amazonEMRConfig.enableEMRDebugging", | |
| "value" : true | |
| }, { | |
| "name" : "amazonEMRConfig.serviceRole", | |
| "value" : "EMR_DefaultRole" | |
| }, { | |
| "name" : "amazonEMRConfig.jobFlowRole", | |
| "value" : "EMR_EC2_DefaultRole" | |
| }, { | |
| "name" : "amazonEMRConfig.visibleToAllUsers", | |
| "value" : true | |
| }, { | |
| "name" : "amazonEMRConfig.ec2SubnetId", | |
| "value" : null | |
| }, { | |
| "name" : "amazonEMRConfig.masterSecurityGroup", | |
| "value" : null | |
| }, { | |
| "name" : "amazonEMRConfig.slaveSecurityGroup", | |
| "value" : null | |
| }, { | |
| "name" : "amazonEMRConfig.instanceCount", | |
| "value" : 2 | |
| }, { | |
| "name" : "amazonEMRConfig.masterInstanceType", | |
| "value" : null | |
| }, { | |
| "name" : "amazonEMRConfig.masterInstanceTypeCustom", | |
| "value" : null | |
| }, { | |
| "name" : "amazonEMRConfig.slaveInstanceType", | |
| "value" : null | |
| }, { | |
| "name" : "amazonEMRConfig.slaveInstanceTypeCustom", | |
| "value" : null | |
| } ], | |
| "uiInfo" : { | |
| "previewConfig" : { | |
| "previewSource" : "CONFIGURED_SOURCE", | |
| "batchSize" : 10, | |
| "timeout" : 30000, | |
| "writeToDestinations" : false, | |
| "executeLifecycleEvents" : false, | |
| "showHeader" : true, | |
| "showFieldType" : true, | |
| "rememberMe" : false | |
| } | |
| }, | |
| "fragments" : [ ], | |
| "stages" : [ { | |
| "instanceName" : "HTTPClient_01", | |
| "library" : "streamsets-datacollector-basic-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_origin_http_HttpClientDSource", | |
| "stageVersion" : "14", | |
| "configuration" : [ { | |
| "name" : "conf.basic.maxBatchSize", | |
| "value" : 1000 | |
| }, { | |
| "name" : "conf.basic.maxWaitTime", | |
| "value" : 2000 | |
| }, { | |
| "name" : "conf.dataFormatConfig.compression", | |
| "value" : "NONE" | |
| }, { | |
| "name" : "conf.dataFormatConfig.filePatternInArchive", | |
| "value" : "*" | |
| }, { | |
| "name" : "conf.dataFormatConfig.charset", | |
| "value" : "UTF-8" | |
| }, { | |
| "name" : "conf.dataFormatConfig.removeCtrlChars", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.textMaxLineLen", | |
| "value" : 1024 | |
| }, { | |
| "name" : "conf.dataFormatConfig.useCustomDelimiter", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.customDelimiter", | |
| "value" : "\\r\\n" | |
| }, { | |
| "name" : "conf.dataFormatConfig.includeCustomDelimiterInTheText", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.jsonContent", | |
| "value" : "MULTIPLE_OBJECTS" | |
| }, { | |
| "name" : "conf.dataFormatConfig.jsonMaxObjectLen", | |
| "value" : 409600 | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvFileFormat", | |
| "value" : "CSV" | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvHeader", | |
| "value" : "NO_HEADER" | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvAllowExtraColumns", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvExtraColumnPrefix", | |
| "value" : "_extra_" | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvMaxObjectLen", | |
| "value" : 1024 | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvCustomDelimiter", | |
| "value" : "|" | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvCustomEscape", | |
| "value" : "\\" | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvCustomQuote", | |
| "value" : "\"" | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvEnableComments", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvCommentMarker", | |
| "value" : "#" | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvIgnoreEmptyLines", | |
| "value" : true | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvRecordType", | |
| "value" : "LIST_MAP" | |
| }, { | |
| "name" : "conf.dataFormatConfig.csvSkipStartLines", | |
| "value" : 0 | |
| }, { | |
| "name" : "conf.dataFormatConfig.parseNull", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.nullConstant", | |
| "value" : "\\\\N" | |
| }, { | |
| "name" : "conf.dataFormatConfig.xmlRecordElement", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.includeFieldXpathAttributes", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.xPathNamespaceContext", | |
| "value" : [ ] | |
| }, { | |
| "name" : "conf.dataFormatConfig.outputFieldAttributes", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.xmlMaxObjectLen", | |
| "value" : 4096 | |
| }, { | |
| "name" : "conf.dataFormatConfig.logMode", | |
| "value" : "COMMON_LOG_FORMAT" | |
| }, { | |
| "name" : "conf.dataFormatConfig.logMaxObjectLen", | |
| "value" : 1024 | |
| }, { | |
| "name" : "conf.dataFormatConfig.retainOriginalLine", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.customLogFormat", | |
| "value" : "%h %l %u %t \"%r\" %>s %b" | |
| }, { | |
| "name" : "conf.dataFormatConfig.regex", | |
| "value" : "^(\\S+) (\\S+) (\\S+) \\[([\\w:/]+\\s[+\\-]\\d{4})\\] \"(\\S+) (\\S+) (\\S+)\" (\\d{3}) (\\d+)" | |
| }, { | |
| "name" : "conf.dataFormatConfig.fieldPathsToGroupName", | |
| "value" : [ { | |
| "fieldPath" : "/", | |
| "group" : 1 | |
| } ] | |
| }, { | |
| "name" : "conf.dataFormatConfig.grokPatternDefinition", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.grokPattern", | |
| "value" : "%{COMMONAPACHELOG}" | |
| }, { | |
| "name" : "conf.dataFormatConfig.onParseError", | |
| "value" : "ERROR" | |
| }, { | |
| "name" : "conf.dataFormatConfig.maxStackTraceLines", | |
| "value" : 50 | |
| }, { | |
| "name" : "conf.dataFormatConfig.enableLog4jCustomLogFormat", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.log4jCustomLogFormat", | |
| "value" : "%r [%t] %-5p %c %x - %m%n" | |
| }, { | |
| "name" : "conf.dataFormatConfig.avroSchemaSource", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.avroSchema", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.schemaRegistryUrls", | |
| "value" : [ ] | |
| }, { | |
| "name" : "conf.dataFormatConfig.schemaLookupMode", | |
| "value" : "SUBJECT" | |
| }, { | |
| "name" : "conf.dataFormatConfig.subject", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.schemaId", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.protoDescriptorFile", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.messageType", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.isDelimited", | |
| "value" : true | |
| }, { | |
| "name" : "conf.dataFormatConfig.binaryMaxObjectLen", | |
| "value" : 1024 | |
| }, { | |
| "name" : "conf.dataFormatConfig.datagramMode", | |
| "value" : "SYSLOG" | |
| }, { | |
| "name" : "conf.dataFormatConfig.typesDbPath", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.convertTime", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.excludeInterval", | |
| "value" : true | |
| }, { | |
| "name" : "conf.dataFormatConfig.authFilePath", | |
| "value" : null | |
| }, { | |
| "name" : "conf.dataFormatConfig.netflowOutputValuesMode", | |
| "value" : "RAW_AND_INTERPRETED" | |
| }, { | |
| "name" : "conf.dataFormatConfig.maxTemplateCacheSize", | |
| "value" : -1 | |
| }, { | |
| "name" : "conf.dataFormatConfig.templateCacheTimeoutMs", | |
| "value" : -1 | |
| }, { | |
| "name" : "conf.dataFormatConfig.netflowOutputValuesModeDatagram", | |
| "value" : "RAW_AND_INTERPRETED" | |
| }, { | |
| "name" : "conf.dataFormatConfig.maxTemplateCacheSizeDatagram", | |
| "value" : -1 | |
| }, { | |
| "name" : "conf.dataFormatConfig.templateCacheTimeoutMsDatagram", | |
| "value" : -1 | |
| }, { | |
| "name" : "conf.dataFormatConfig.wholeFileMaxObjectLen", | |
| "value" : 8192 | |
| }, { | |
| "name" : "conf.dataFormatConfig.rateLimit", | |
| "value" : "-1" | |
| }, { | |
| "name" : "conf.dataFormatConfig.verifyChecksum", | |
| "value" : false | |
| }, { | |
| "name" : "conf.dataFormatConfig.excelHeader", | |
| "value" : null | |
| }, { | |
| "name" : "conf.resourceUrl", | |
| "value" : "https://api.box.com/2.0/folders/0" | |
| }, { | |
| "name" : "conf.headers", | |
| "value" : [ ] | |
| }, { | |
| "name" : "conf.httpMethod", | |
| "value" : "GET" | |
| }, { | |
| "name" : "conf.timeZoneID", | |
| "value" : "UTC" | |
| }, { | |
| "name" : "conf.requestBody", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.defaultRequestContentType", | |
| "value" : "application/json" | |
| }, { | |
| "name" : "conf.client.transferEncoding", | |
| "value" : "BUFFERED" | |
| }, { | |
| "name" : "conf.client.httpCompression", | |
| "value" : "NONE" | |
| }, { | |
| "name" : "conf.client.connectTimeoutMillis", | |
| "value" : 0 | |
| }, { | |
| "name" : "conf.client.readTimeoutMillis", | |
| "value" : 0 | |
| }, { | |
| "name" : "conf.client.authType", | |
| "value" : "NONE" | |
| }, { | |
| "name" : "conf.client.useOAuth2", | |
| "value" : true | |
| }, { | |
| "name" : "conf.client.oauth.consumerKey", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.client.oauth.consumerSecret", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.client.oauth.token", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.client.oauth.tokenSecret", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.client.oauth2.credentialsGrantType", | |
| "value" : "JWT" | |
| }, { | |
| "name" : "conf.client.oauth2.tokenUrl", | |
| "value" : "https://api.box.com/oauth2/token" | |
| }, { | |
| "name" : "conf.client.oauth2.clientId", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.client.oauth2.clientSecret", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.client.oauth2.username", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.client.oauth2.password", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.client.oauth2.resourceOwnerClientId", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.client.oauth2.resourceOwnerClientSecret", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.client.oauth2.algorithm", | |
| "value" : "RS256" | |
| }, { | |
| "name" : "conf.client.oauth2.key", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.client.oauth2.jwtClaims", | |
| "value" : "{\n \"iss\":\"d0t38y0cuhr4gv4eqtczjszxwv6f0627\",\n \"sub\":\"227817445\",\n \"box_sub_type\":\"enterprise\",\n \"aud\":\"https://api.box.com/oauth2/token\",\n \"exp\":200000,\n \"jti\":\"kothinakodaka110\",\n \"iat\":1516239033\n}" | |
| }, { | |
| "name" : "conf.client.oauth2.transferEncoding", | |
| "value" : "BUFFERED" | |
| }, { | |
| "name" : "conf.client.oauth2.additionalValues", | |
| "value" : [ { | |
| "key" : "grant_type", | |
| "value" : "" | |
| }, { | |
| "key" : "client_id", | |
| "value" : "" | |
| }, { | |
| "key" : "client_secret", | |
| "value" : "" | |
| } ] | |
| }, { | |
| "name" : "conf.client.basicAuth.username", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.client.basicAuth.password", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.client.useProxy", | |
| "value" : false | |
| }, { | |
| "name" : "conf.client.proxy.uri", | |
| "value" : null | |
| }, { | |
| "name" : "conf.client.proxy.username", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.client.proxy.password", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.client.tlsConfig.tlsEnabled", | |
| "value" : false | |
| }, { | |
| "name" : "conf.client.tlsConfig.keyStoreFilePath", | |
| "value" : null | |
| }, { | |
| "name" : "conf.client.tlsConfig.keyStoreType", | |
| "value" : "JKS" | |
| }, { | |
| "name" : "conf.client.tlsConfig.keyStorePassword", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.client.tlsConfig.keyStoreAlgorithm", | |
| "value" : "SunX509" | |
| }, { | |
| "name" : "conf.client.tlsConfig.trustStoreFilePath", | |
| "value" : null | |
| }, { | |
| "name" : "conf.client.tlsConfig.trustStoreType", | |
| "value" : "JKS" | |
| }, { | |
| "name" : "conf.client.tlsConfig.trustStorePassword", | |
| "value" : "" | |
| }, { | |
| "name" : "conf.client.tlsConfig.trustStoreAlgorithm", | |
| "value" : "SunX509" | |
| }, { | |
| "name" : "conf.client.tlsConfig.useDefaultProtocols", | |
| "value" : true | |
| }, { | |
| "name" : "conf.client.tlsConfig.protocols", | |
| "value" : [ ] | |
| }, { | |
| "name" : "conf.client.tlsConfig.useDefaultCiperSuites", | |
| "value" : true | |
| }, { | |
| "name" : "conf.client.tlsConfig.cipherSuites", | |
| "value" : [ ] | |
| }, { | |
| "name" : "conf.client.requestLoggingConfig.enableRequestLogging", | |
| "value" : true | |
| }, { | |
| "name" : "conf.client.requestLoggingConfig.logLevel", | |
| "value" : "ALL" | |
| }, { | |
| "name" : "conf.client.requestLoggingConfig.verbosity", | |
| "value" : "PAYLOAD_ANY" | |
| }, { | |
| "name" : "conf.client.requestLoggingConfig.maxEntitySize", | |
| "value" : 0 | |
| }, { | |
| "name" : "conf.httpMode", | |
| "value" : "BATCH" | |
| }, { | |
| "name" : "conf.pollingInterval", | |
| "value" : 5000 | |
| }, { | |
| "name" : "conf.dataFormat", | |
| "value" : "JSON" | |
| }, { | |
| "name" : "conf.responseStatusActionConfigs", | |
| "value" : [ { | |
| "statusCode" : 500, | |
| "action" : "RETRY_EXPONENTIAL_BACKOFF", | |
| "backoffInterval" : 1000, | |
| "maxNumRetries" : 10 | |
| } ] | |
| }, { | |
| "name" : "conf.responseTimeoutActionConfig.action", | |
| "value" : "RETRY_IMMEDIATELY" | |
| }, { | |
| "name" : "conf.responseTimeoutActionConfig.backoffInterval", | |
| "value" : 1000 | |
| }, { | |
| "name" : "conf.responseTimeoutActionConfig.maxNumRetries", | |
| "value" : 10 | |
| }, { | |
| "name" : "conf.pagination.mode", | |
| "value" : "NONE" | |
| }, { | |
| "name" : "conf.pagination.nextPageFieldPath", | |
| "value" : null | |
| }, { | |
| "name" : "conf.pagination.stopCondition", | |
| "value" : null | |
| }, { | |
| "name" : "conf.pagination.startAt", | |
| "value" : 0 | |
| }, { | |
| "name" : "conf.pagination.resultFieldPath", | |
| "value" : null | |
| }, { | |
| "name" : "conf.pagination.keepAllFields", | |
| "value" : false | |
| }, { | |
| "name" : "conf.pagination.rateLimit", | |
| "value" : 2000 | |
| }, { | |
| "name" : "stageOnRecordError", | |
| "value" : "TO_ERROR" | |
| }, { | |
| "name" : "conf.dataFormatConfig.multiCharacterFieldDelimiter", | |
| "value" : "||" | |
| }, { | |
| "name" : "conf.dataFormatConfig.multiCharacterLineDelimiter", | |
| "value" : "${str:unescapeJava('\\\\n')}" | |
| } ], | |
| "uiInfo" : { | |
| "description" : "", | |
| "label" : "HTTP Client 1", | |
| "xPos" : 44, | |
| "yPos" : 27, | |
| "stageType" : "SOURCE" | |
| }, | |
| "inputLanes" : [ ], | |
| "outputLanes" : [ "HTTPClient_01OutputLane15336225894850" ], | |
| "eventLanes" : [ ], | |
| "services" : [ ] | |
| }, { | |
| "instanceName" : "Trash_01", | |
| "library" : "streamsets-datacollector-basic-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_destination_devnull_NullDTarget", | |
| "stageVersion" : "1", | |
| "configuration" : [ ], | |
| "uiInfo" : { | |
| "description" : "", | |
| "label" : "Trash 1", | |
| "xPos" : 495, | |
| "yPos" : 48, | |
| "stageType" : "TARGET" | |
| }, | |
| "inputLanes" : [ "HTTPClient_01OutputLane15336225894850" ], | |
| "outputLanes" : [ ], | |
| "eventLanes" : [ ], | |
| "services" : [ ] | |
| } ], | |
| "errorStage" : { | |
| "instanceName" : "Discard_ErrorStage", | |
| "library" : "streamsets-datacollector-basic-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget", | |
| "stageVersion" : "1", | |
| "configuration" : [ ], | |
| "uiInfo" : { | |
| "description" : "", | |
| "label" : "Error Records - Discard", | |
| "xPos" : 280, | |
| "yPos" : 50, | |
| "stageType" : "TARGET" | |
| }, | |
| "inputLanes" : [ ], | |
| "outputLanes" : [ ], | |
| "eventLanes" : [ ], | |
| "services" : [ ] | |
| }, | |
| "info" : { | |
| "pipelineId" : "personalbox4f3590a2-1b4e-40ac-beae-600e976ef496", | |
| "title" : "personal_box", | |
| "description" : "", | |
| "created" : 1565594966124, | |
| "lastModified" : 1566310554523, | |
| "creator" : "admin", | |
| "lastModifier" : "admin", | |
| "lastRev" : "0", | |
| "uuid" : "043a4e85-7989-4761-90de-e68401257e9e", | |
| "valid" : true, | |
| "metadata" : { | |
| "labels" : [ "WWSQA" ] | |
| }, | |
| "name" : "personalbox4f3590a2-1b4e-40ac-beae-600e976ef496", | |
| "sdcVersion" : "3.8.0", | |
| "sdcId" : "a91a3e9a-56a6-11e9-84dc-5b0486474db2" | |
| }, | |
| "metadata" : { | |
| "labels" : [ "WWSQA" ] | |
| }, | |
| "statsAggregatorStage" : { | |
| "instanceName" : "statsAggregatorStageInstance", | |
| "library" : "streamsets-datacollector-basic-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_destination_devnull_StatsDpmDirectlyDTarget", | |
| "stageVersion" : "1", | |
| "configuration" : [ ], | |
| "uiInfo" : { | |
| "stageType" : "TARGET", | |
| "label" : "Stats Aggregator -Write to Control Hub directly" | |
| }, | |
| "inputLanes" : [ ], | |
| "outputLanes" : [ ], | |
| "eventLanes" : [ ], | |
| "services" : [ ] | |
| }, | |
| "startEventStages" : [ { | |
| "instanceName" : "Discard_StartEventStage", | |
| "library" : "streamsets-datacollector-basic-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget", | |
| "stageVersion" : "1", | |
| "configuration" : [ ], | |
| "uiInfo" : { | |
| "description" : "", | |
| "label" : "Start Event - Discard", | |
| "xPos" : 280, | |
| "yPos" : 50, | |
| "stageType" : "TARGET" | |
| }, | |
| "inputLanes" : [ ], | |
| "outputLanes" : [ ], | |
| "eventLanes" : [ ], | |
| "services" : [ ] | |
| } ], | |
| "stopEventStages" : [ { | |
| "instanceName" : "Discard_StopEventStage", | |
| "library" : "streamsets-datacollector-basic-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget", | |
| "stageVersion" : "1", | |
| "configuration" : [ ], | |
| "uiInfo" : { | |
| "description" : "", | |
| "label" : "Stop Event - Discard", | |
| "xPos" : 280, | |
| "yPos" : 50, | |
| "stageType" : "TARGET" | |
| }, | |
| "inputLanes" : [ ], | |
| "outputLanes" : [ ], | |
| "eventLanes" : [ ], | |
| "services" : [ ] | |
| } ], | |
| "testOriginStage" : { | |
| "instanceName" : "com_streamsets_pipeline_stage_devtest_rawdata_RawDataDSource_TestOriginStage", | |
| "library" : "streamsets-datacollector-dev-lib", | |
| "stageName" : "com_streamsets_pipeline_stage_devtest_rawdata_RawDataDSource", | |
| "stageVersion" : "3", | |
| "configuration" : [ { | |
| "name" : "rawData", | |
| "value" : "{\n \"f1\": \"abc\",\n \"f2\": \"xyz\",\n \"f3\": \"lmn\"\n}" | |
| }, { | |
| "name" : "stopAfterFirstBatch", | |
| "value" : false | |
| }, { | |
| "name" : "eventData", | |
| "value" : null | |
| }, { | |
| "name" : "stageOnRecordError", | |
| "value" : "TO_ERROR" | |
| } ], | |
| "uiInfo" : { | |
| "stageType" : "SOURCE", | |
| "label" : "Test Origin - Dev Raw Data Source" | |
| }, | |
| "inputLanes" : [ ], | |
| "outputLanes" : [ "com_streamsets_pipeline_stage_devtest_rawdata_RawDataDSource_TestOriginStageOutputLane1" ], | |
| "eventLanes" : [ ], | |
| "services" : [ { | |
| "service" : "com.streamsets.pipeline.api.service.dataformats.DataFormatParserService", | |
| "serviceVersion" : 1, | |
| "configuration" : [ { | |
| "name" : "displayFormats", | |
| "value" : "DELIMITED,JSON,LOG,SDC_JSON,TEXT,XML" | |
| }, { | |
| "name" : "dataFormat", | |
| "value" : "JSON" | |
| }, { | |
| "name" : "dataFormatConfig.compression", | |
| "value" : "NONE" | |
| }, { | |
| "name" : "dataFormatConfig.filePatternInArchive", | |
| "value" : "*" | |
| }, { | |
| "name" : "dataFormatConfig.charset", | |
| "value" : "UTF-8" | |
| }, { | |
| "name" : "dataFormatConfig.removeCtrlChars", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.textMaxLineLen", | |
| "value" : 1024 | |
| }, { | |
| "name" : "dataFormatConfig.useCustomDelimiter", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.customDelimiter", | |
| "value" : "\\r\\n" | |
| }, { | |
| "name" : "dataFormatConfig.includeCustomDelimiterInTheText", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.jsonContent", | |
| "value" : "MULTIPLE_OBJECTS" | |
| }, { | |
| "name" : "dataFormatConfig.jsonMaxObjectLen", | |
| "value" : 4096 | |
| }, { | |
| "name" : "dataFormatConfig.csvFileFormat", | |
| "value" : "CSV" | |
| }, { | |
| "name" : "dataFormatConfig.csvHeader", | |
| "value" : "NO_HEADER" | |
| }, { | |
| "name" : "dataFormatConfig.csvAllowExtraColumns", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.csvExtraColumnPrefix", | |
| "value" : "_extra_" | |
| }, { | |
| "name" : "dataFormatConfig.csvMaxObjectLen", | |
| "value" : 1024 | |
| }, { | |
| "name" : "dataFormatConfig.csvCustomDelimiter", | |
| "value" : "|" | |
| }, { | |
| "name" : "dataFormatConfig.multiCharacterFieldDelimiter", | |
| "value" : "||" | |
| }, { | |
| "name" : "dataFormatConfig.multiCharacterLineDelimiter", | |
| "value" : "${str:unescapeJava('\\\\n')}" | |
| }, { | |
| "name" : "dataFormatConfig.csvCustomEscape", | |
| "value" : "\\" | |
| }, { | |
| "name" : "dataFormatConfig.csvCustomQuote", | |
| "value" : "\"" | |
| }, { | |
| "name" : "dataFormatConfig.csvEnableComments", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.csvCommentMarker", | |
| "value" : "#" | |
| }, { | |
| "name" : "dataFormatConfig.csvIgnoreEmptyLines", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.csvRecordType", | |
| "value" : "LIST_MAP" | |
| }, { | |
| "name" : "dataFormatConfig.csvSkipStartLines", | |
| "value" : 0 | |
| }, { | |
| "name" : "dataFormatConfig.parseNull", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.nullConstant", | |
| "value" : "\\\\N" | |
| }, { | |
| "name" : "dataFormatConfig.xmlRecordElement", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.includeFieldXpathAttributes", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.xPathNamespaceContext", | |
| "value" : [ ] | |
| }, { | |
| "name" : "dataFormatConfig.outputFieldAttributes", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.xmlMaxObjectLen", | |
| "value" : 4096 | |
| }, { | |
| "name" : "dataFormatConfig.logMode", | |
| "value" : "COMMON_LOG_FORMAT" | |
| }, { | |
| "name" : "dataFormatConfig.logMaxObjectLen", | |
| "value" : 1024 | |
| }, { | |
| "name" : "dataFormatConfig.retainOriginalLine", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.customLogFormat", | |
| "value" : "%h %l %u %t \"%r\" %>s %b" | |
| }, { | |
| "name" : "dataFormatConfig.regex", | |
| "value" : "^(\\S+) (\\S+) (\\S+) \\[([\\w:/]+\\s[+\\-]\\d{4})\\] \"(\\S+) (\\S+) (\\S+)\" (\\d{3}) (\\d+)" | |
| }, { | |
| "name" : "dataFormatConfig.fieldPathsToGroupName", | |
| "value" : [ { | |
| "fieldPath" : "/", | |
| "group" : 1 | |
| } ] | |
| }, { | |
| "name" : "dataFormatConfig.grokPatternDefinition", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.grokPattern", | |
| "value" : "%{COMMONAPACHELOG}" | |
| }, { | |
| "name" : "dataFormatConfig.onParseError", | |
| "value" : "ERROR" | |
| }, { | |
| "name" : "dataFormatConfig.maxStackTraceLines", | |
| "value" : 50 | |
| }, { | |
| "name" : "dataFormatConfig.enableLog4jCustomLogFormat", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.log4jCustomLogFormat", | |
| "value" : "%r [%t] %-5p %c %x - %m%n" | |
| }, { | |
| "name" : "dataFormatConfig.avroSchemaSource", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.avroSchema", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.schemaRegistryUrls", | |
| "value" : [ ] | |
| }, { | |
| "name" : "dataFormatConfig.schemaLookupMode", | |
| "value" : "SUBJECT" | |
| }, { | |
| "name" : "dataFormatConfig.subject", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.schemaId", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.protoDescriptorFile", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.messageType", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.isDelimited", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.binaryMaxObjectLen", | |
| "value" : 1024 | |
| }, { | |
| "name" : "dataFormatConfig.datagramMode", | |
| "value" : "SYSLOG" | |
| }, { | |
| "name" : "dataFormatConfig.typesDbPath", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.convertTime", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.excludeInterval", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.authFilePath", | |
| "value" : null | |
| }, { | |
| "name" : "dataFormatConfig.netflowOutputValuesMode", | |
| "value" : "RAW_AND_INTERPRETED" | |
| }, { | |
| "name" : "dataFormatConfig.maxTemplateCacheSize", | |
| "value" : -1 | |
| }, { | |
| "name" : "dataFormatConfig.templateCacheTimeoutMs", | |
| "value" : -1 | |
| }, { | |
| "name" : "dataFormatConfig.netflowOutputValuesModeDatagram", | |
| "value" : "RAW_AND_INTERPRETED" | |
| }, { | |
| "name" : "dataFormatConfig.maxTemplateCacheSizeDatagram", | |
| "value" : -1 | |
| }, { | |
| "name" : "dataFormatConfig.templateCacheTimeoutMsDatagram", | |
| "value" : -1 | |
| }, { | |
| "name" : "dataFormatConfig.wholeFileMaxObjectLen", | |
| "value" : 8192 | |
| }, { | |
| "name" : "dataFormatConfig.rateLimit", | |
| "value" : "-1" | |
| }, { | |
| "name" : "dataFormatConfig.verifyChecksum", | |
| "value" : false | |
| }, { | |
| "name" : "dataFormatConfig.excelHeader", | |
| "value" : null | |
| } ] | |
| } ] | |
| }, | |
| "valid" : true, | |
| "issues" : { | |
| "issueCount" : 0, | |
| "stageIssues" : { }, | |
| "pipelineIssues" : [ ] | |
| }, | |
| "previewable" : true | |
| }, | |
| "pipelineRules" : { | |
| "schemaVersion" : 3, | |
| "version" : 2, | |
| "metricsRuleDefinitions" : [ { | |
| "id" : "badRecordsAlertID", | |
| "alertText" : "High incidence of Error Records", | |
| "metricId" : "pipeline.batchErrorRecords.counter", | |
| "metricType" : "COUNTER", | |
| "metricElement" : "COUNTER_COUNT", | |
| "condition" : "${value() > 100}", | |
| "sendEmail" : false, | |
| "enabled" : false, | |
| "timestamp" : 1563515361775, | |
| "valid" : true | |
| }, { | |
| "id" : "stageErrorAlertID", | |
| "alertText" : "High incidence of Stage Errors", | |
| "metricId" : "pipeline.batchErrorMessages.counter", | |
| "metricType" : "COUNTER", | |
| "metricElement" : "COUNTER_COUNT", | |
| "condition" : "${value() > 100}", | |
| "sendEmail" : false, | |
| "enabled" : false, | |
| "timestamp" : 1563515361775, | |
| "valid" : true | |
| }, { | |
| "id" : "idleGaugeID", | |
| "alertText" : "Pipeline is Idle", | |
| "metricId" : "RuntimeStatsGauge.gauge", | |
| "metricType" : "GAUGE", | |
| "metricElement" : "TIME_OF_LAST_RECEIVED_RECORD", | |
| "condition" : "${time:now() - value() > 120000}", | |
| "sendEmail" : false, | |
| "enabled" : false, | |
| "timestamp" : 1563515361775, | |
| "valid" : true | |
| }, { | |
| "id" : "batchTimeAlertID", | |
| "alertText" : "Batch taking more time to process", | |
| "metricId" : "RuntimeStatsGauge.gauge", | |
| "metricType" : "GAUGE", | |
| "metricElement" : "CURRENT_BATCH_AGE", | |
| "condition" : "${value() > 200}", | |
| "sendEmail" : false, | |
| "enabled" : false, | |
| "timestamp" : 1563515361775, | |
| "valid" : true | |
| } ], | |
| "dataRuleDefinitions" : [ ], | |
| "driftRuleDefinitions" : [ ], | |
| "uuid" : "62055883-a929-47fd-964f-99e826ef369f", | |
| "configuration" : [ { | |
| "name" : "emailIDs", | |
| "value" : [ ] | |
| }, { | |
| "name" : "webhookConfigs", | |
| "value" : [ ] | |
| } ], | |
| "ruleIssues" : [ ], | |
| "configIssues" : [ ] | |
| }, | |
| "libraryDefinitions" : null | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment