Skip to content

Instantly share code, notes, and snippets.

@j3works

j3works/Formatted MM_dd_YYYY HH_mm_ss date to ....json

Last active May 3, 2020 03:09
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save j3works/952878b8a20ca8e6f06b5d7b86e624e1 to your computer and use it in GitHub Desktop.
Save j3works/952878b8a20ca8e6f06b5d7b86e624e1 to your computer and use it in GitHub Desktop.
Download ZIP
Couple examples of converting a date string into a different format and into UNIX time (epoch)
Raw
Formatted MM_dd_YYYY HH_mm_ss date to ....json
{
"pipelineConfig" : {
"schemaVersion" : 6,
"version" : 16,
"pipelineId" : "FormattedMMddYYYYHHmmssdateto677507fb-d4d8-44bf-b01f-98350efbf5de",
"title" : "Formatted MM_dd_YYYY HH_mm_ss date to ...",
"description" : "",
"uuid" : "ddf1c8ab-07dd-4318-bf7c-30fa2070ccc4",
"configuration" : [ {
"name" : "executionMode",
"value" : "STANDALONE"
}, {
"name" : "edgeHttpUrl",
"value" : "http://localhost:18633"
}, {
"name" : "deliveryGuarantee",
"value" : "AT_LEAST_ONCE"
}, {
"name" : "testOriginStage",
"value" : "streamsets-datacollector-dev-lib::com_streamsets_pipeline_stage_devtest_rawdata_RawDataDSource::3"
}, {
"name" : "startEventStage",
"value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1"
}, {
"name" : "stopEventStage",
"value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1"
}, {
"name" : "shouldRetry",
"value" : false
}, {
"name" : "triggerInterval",
"value" : 2000
}, {
"name" : "retryAttempts",
"value" : -1
}, {
"name" : "ludicrousMode",
"value" : false
}, {
"name" : "ludicrousModeInputCount",
"value" : false
}, {
"name" : "advancedErrorHandling",
"value" : false
}, {
"name" : "notifyOnStates",
"value" : [ "RUN_ERROR", "STOPPED", "FINISHED" ]
}, {
"name" : "emailIDs",
"value" : [ ]
}, {
"name" : "constants",
"value" : [ ]
}, {
"name" : "badRecordsHandling",
"value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1"
}, {
"name" : "errorRecordPolicy",
"value" : "ORIGINAL_RECORD"
}, {
"name" : "statsAggregatorStage",
"value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_StatsDpmDirectlyDTarget::1"
}, {
"name" : "workerCount",
"value" : 0
}, {
"name" : "clusterSlaveMemory",
"value" : 2048
}, {
"name" : "clusterSlaveJavaOpts",
"value" : "-XX:+UseConcMarkSweepGC -XX:+UseParNewGC -Dlog4j.debug"
}, {
"name" : "clusterLauncherEnv",
"value" : [ ]
}, {
"name" : "mesosDispatcherURL",
"value" : null
}, {
"name" : "logLevel",
"value" : "INFO"
}, {
"name" : "hdfsS3ConfDir",
"value" : null
}, {
"name" : "rateLimit",
"value" : 0
}, {
"name" : "maxRunners",
"value" : 0
}, {
"name" : "shouldCreateFailureSnapshot",
"value" : true
}, {
"name" : "runnerIdleTIme",
"value" : 60
}, {
"name" : "webhookConfigs",
"value" : [ ]
}, {
"name" : "sparkConfigs",
"value" : [ ]
}, {
"name" : "preprocessScript",
"value" : "/*\nThe following script define a method\nthat increments an integer by 1 \nand registers it as a UDF with \nthe SparkSession, which can be accessed\nusing the variable named \"spark\":\ndef inc(i: Integer): Integer = {\n i + 1\n}\nspark.udf.register (\"inc\", inc _)\n\n*/"
}, {
"name" : "clusterConfig.clusterType",
"value" : "LOCAL"
}, {
"name" : "clusterConfig.sparkMasterUrl",
"value" : "local[*]"
}, {
"name" : "clusterConfig.deployMode",
"value" : "CLIENT"
}, {
"name" : "clusterConfig.hadoopUserName",
"value" : null
}, {
"name" : "clusterConfig.sparkAppName",
"value" : "${pipeline:title()}"
}, {
"name" : "clusterConfig.stagingDir",
"value" : "/streamsets"
}, {
"name" : "clusterConfig.useYarnKerberosKeytab",
"value" : false
}, {
"name" : "clusterConfig.yarnKerberosKeytabSource",
"value" : "PROPERTIES_FILE"
}, {
"name" : "clusterConfig.yarnKerberosKeytab",
"value" : null
}, {
"name" : "clusterConfig.yarnKerberosPrincipal",
"value" : "name@DOMAIN"
}, {
"name" : "databricksConfig.baseUrl",
"value" : null
}, {
"name" : "databricksConfig.credentialType",
"value" : null
}, {
"name" : "databricksConfig.username",
"value" : null
}, {
"name" : "databricksConfig.password",
"value" : null
}, {
"name" : "databricksConfig.token",
"value" : null
}, {
"name" : "databricksConfig.provisionNewCluster",
"value" : true
}, {
"name" : "databricksConfig.clusterId",
"value" : null
}, {
"name" : "databricksConfig.clusterConfig",
"value" : "{\n \"num_workers\": 8,\n \"spark_version\": \"5.3.x-scala2.11\",\n \"node_type_id\": \"i3.xlarge\"\n}"
}, {
"name" : "databricksConfig.terminateCluster",
"value" : false
}, {
"name" : "livyConfig.baseUrl",
"value" : "https://localhost:30443/gateway/default/livy/v1/"
}, {
"name" : "livyConfig.username",
"value" : null
}, {
"name" : "livyConfig.password",
"value" : null
}, {
"name" : "amazonEMRConfig.userRegion",
"value" : null
}, {
"name" : "amazonEMRConfig.userRegionCustom",
"value" : null
}, {
"name" : "amazonEMRConfig.accessKey",
"value" : null
}, {
"name" : "amazonEMRConfig.secretKey",
"value" : null
}, {
"name" : "amazonEMRConfig.s3StagingUri",
"value" : null
}, {
"name" : "amazonEMRConfig.provisionNewCluster",
"value" : false
}, {
"name" : "amazonEMRConfig.clusterId",
"value" : null
}, {
"name" : "amazonEMRConfig.clusterPrefix",
"value" : null
}, {
"name" : "amazonEMRConfig.terminateCluster",
"value" : false
}, {
"name" : "amazonEMRConfig.loggingEnabled",
"value" : true
}, {
"name" : "amazonEMRConfig.s3LogUri",
"value" : null
}, {
"name" : "amazonEMRConfig.enableEMRDebugging",
"value" : true
}, {
"name" : "amazonEMRConfig.serviceRole",
"value" : "EMR_DefaultRole"
}, {
"name" : "amazonEMRConfig.jobFlowRole",
"value" : "EMR_EC2_DefaultRole"
}, {
"name" : "amazonEMRConfig.visibleToAllUsers",
"value" : true
}, {
"name" : "amazonEMRConfig.ec2SubnetId",
"value" : null
}, {
"name" : "amazonEMRConfig.masterSecurityGroup",
"value" : null
}, {
"name" : "amazonEMRConfig.slaveSecurityGroup",
"value" : null
}, {
"name" : "amazonEMRConfig.instanceCount",
"value" : 2
}, {
"name" : "amazonEMRConfig.masterInstanceType",
"value" : null
}, {
"name" : "amazonEMRConfig.masterInstanceTypeCustom",
"value" : null
}, {
"name" : "amazonEMRConfig.slaveInstanceType",
"value" : null
}, {
"name" : "amazonEMRConfig.slaveInstanceTypeCustom",
"value" : null
} ],
"uiInfo" : {
"previewConfig" : {
"previewSource" : "CONFIGURED_SOURCE",
"batchSize" : 10,
"timeout" : 30000,
"writeToDestinations" : false,
"executeLifecycleEvents" : false,
"showHeader" : false,
"showFieldType" : true,
"rememberMe" : true
}
},
"fragments" : [ ],
"stages" : [ {
"instanceName" : "DevDataGenerator_01",
"library" : "streamsets-datacollector-dev-lib",
"stageName" : "com_streamsets_pipeline_stage_devtest_RandomDataGeneratorSource",
"stageVersion" : "5",
"configuration" : [ {
"name" : "dataGenConfigs",
"value" : [ {
"type" : "DATETIME",
"precision" : 10,
"scale" : 2,
"field" : "timestamp"
} ]
}, {
"name" : "rootFieldType",
"value" : "MAP"
}, {
"name" : "headerAttributes",
"value" : [ ]
}, {
"name" : "delay",
"value" : 1000
}, {
"name" : "batchSize",
"value" : 1000
}, {
"name" : "numThreads",
"value" : 1
}, {
"name" : "eventName",
"value" : "generated-event"
}, {
"name" : "stageOnRecordError",
"value" : "TO_ERROR"
} ],
"uiInfo" : {
"yPos" : 50,
"stageType" : "SOURCE",
"icon" : "dev.png",
"description" : "",
"label" : "Random dates",
"xPos" : 60
},
"inputLanes" : [ ],
"outputLanes" : [ "DevDataGenerator_01OutputLane15884462298650" ],
"eventLanes" : [ ],
"services" : [ ]
}, {
"instanceName" : "FieldTypeConverter_01",
"library" : "streamsets-datacollector-basic-lib",
"stageName" : "com_streamsets_pipeline_stage_processor_fieldtypeconverter_FieldTypeConverterDProcessor",
"stageVersion" : "2",
"configuration" : [ {
"name" : "convertBy",
"value" : "BY_FIELD"
}, {
"name" : "fieldTypeConverterConfigs",
"value" : [ {
"fields" : [ "/timestamp" ],
"targetType" : "STRING",
"treatInputFieldAsDate" : false,
"dataLocale" : "en,US",
"scale" : -1,
"decimalScaleRoundingStrategy" : "ROUND_UNNECESSARY",
"dateFormat" : "OTHER",
"zonedDateTimeFormat" : "ISO_ZONED_DATE_TIME",
"encoding" : "UTF-8",
"otherDateFormat" : "MM/dd/yyyy HH:mm:ss"
} ]
}, {
"name" : "wholeTypeConverterConfigs",
"value" : [ {
"sourceType" : "INTEGER",
"targetType" : "INTEGER",
"treatInputFieldAsDate" : false,
"dataLocale" : "en,US",
"scale" : -1,
"decimalScaleRoundingStrategy" : "ROUND_UNNECESSARY",
"dateFormat" : "YYYY_MM_DD",
"zonedDateTimeFormat" : "ISO_ZONED_DATE_TIME",
"encoding" : "UTF-8"
} ]
}, {
"name" : "stageOnRecordError",
"value" : "TO_ERROR"
}, {
"name" : "stageRequiredFields",
"value" : [ ]
}, {
"name" : "stageRecordPreconditions",
"value" : [ ]
} ],
"uiInfo" : {
"yPos" : 50,
"stageType" : "PROCESSOR",
"icon" : "converter.png",
"description" : "",
"label" : "Convert dates to mm/dd/yyyy strings",
"xPos" : 280
},
"inputLanes" : [ "DevDataGenerator_01OutputLane15884462298650" ],
"outputLanes" : [ "FieldTypeConverter_01OutputLane15884464073820" ],
"eventLanes" : [ ],
"services" : [ ]
}, {
"instanceName" : "ExpressionEvaluator_01",
"library" : "streamsets-datacollector-basic-lib",
"stageName" : "com_streamsets_pipeline_stage_processor_expression_ExpressionDProcessor",
"stageVersion" : "2",
"configuration" : [ {
"name" : "expressionProcessorConfigs",
"value" : [ {
"fieldToSet" : "/to_yyyy_mm_dd_hh_mm_ss",
"expression" : "${time:extractStringFromDate(time:createDateFromStringTZ(record:value('/timestamp'),'UTC','MM/dd/yyyy HH:mm:ss'),'yyyy-MM-dd HH:mm:ss')}"
}, {
"fieldToSet" : "/to_epoch",
"expression" : "${time:dateTimeToMilliseconds(time:extractDateFromString(record:value('/timestamp'),'MM/dd/yyyy HH:mm:ss'))}"
} ]
}, {
"name" : "headerAttributeConfigs",
"value" : [ { } ]
}, {
"name" : "fieldAttributeConfigs",
"value" : [ {
"fieldToSet" : "/"
} ]
}, {
"name" : "stageOnRecordError",
"value" : "TO_ERROR"
}, {
"name" : "stageRequiredFields",
"value" : [ ]
}, {
"name" : "stageRecordPreconditions",
"value" : [ ]
} ],
"uiInfo" : {
"yPos" : 50,
"stageType" : "PROCESSOR",
"icon" : "expression.png",
"description" : "",
"label" : "mm/dd/yyyy Format to",
"xPos" : 499
},
"inputLanes" : [ "FieldTypeConverter_01OutputLane15884464073820" ],
"outputLanes" : [ "ExpressionEvaluator_01OutputLane15884465930380" ],
"eventLanes" : [ ],
"services" : [ ]
}, {
"instanceName" : "FieldTypeConverter_02",
"library" : "streamsets-datacollector-basic-lib",
"stageName" : "com_streamsets_pipeline_stage_processor_fieldtypeconverter_FieldTypeConverterDProcessor",
"stageVersion" : "2",
"configuration" : [ {
"name" : "convertBy",
"value" : "BY_FIELD"
}, {
"name" : "fieldTypeConverterConfigs",
"value" : [ {
"fields" : [ "/timestamp" ],
"targetType" : "STRING",
"treatInputFieldAsDate" : false,
"dataLocale" : "en,US",
"scale" : -1,
"decimalScaleRoundingStrategy" : "ROUND_UNNECESSARY",
"dateFormat" : "OTHER",
"zonedDateTimeFormat" : "ISO_ZONED_DATE_TIME",
"encoding" : "UTF-8",
"otherDateFormat" : "yyyy-MM-dd HH:mm:ss"
} ]
}, {
"name" : "wholeTypeConverterConfigs",
"value" : [ {
"sourceType" : "INTEGER",
"targetType" : "INTEGER",
"treatInputFieldAsDate" : false,
"dataLocale" : "en,US",
"scale" : -1,
"decimalScaleRoundingStrategy" : "ROUND_UNNECESSARY",
"dateFormat" : "YYYY_MM_DD",
"zonedDateTimeFormat" : "ISO_ZONED_DATE_TIME",
"encoding" : "UTF-8"
} ]
}, {
"name" : "stageOnRecordError",
"value" : "TO_ERROR"
}, {
"name" : "stageRequiredFields",
"value" : [ ]
}, {
"name" : "stageRecordPreconditions",
"value" : [ ]
} ],
"uiInfo" : {
"yPos" : 200,
"stageType" : "PROCESSOR",
"icon" : "converter.png",
"description" : "",
"label" : "Convert dates to yyyy-mm-dd strings",
"xPos" : 280
},
"inputLanes" : [ "DevDataGenerator_01OutputLane15884462298650" ],
"outputLanes" : [ "FieldTypeConverter_02OutputLane15884749055110" ],
"eventLanes" : [ ],
"services" : [ ]
}, {
"instanceName" : "ExpressionEvaluator_02",
"library" : "streamsets-datacollector-basic-lib",
"stageName" : "com_streamsets_pipeline_stage_processor_expression_ExpressionDProcessor",
"stageVersion" : "2",
"configuration" : [ {
"name" : "expressionProcessorConfigs",
"value" : [ {
"fieldToSet" : "/to_yyyy_mm_dd_hh_mm_ss_2",
"expression" : "${time:extractStringFromDate(time:createDateFromStringTZ(record:value('/timestamp'),'UTC','yyyy-MM-dd HH:mm:ss'),'yyyy-MM-dd HH:mm:ss')}"
}, {
"fieldToSet" : "/to_epoch_2",
"expression" : "${time:dateTimeToMilliseconds(time:extractDateFromString(record:value('/timestamp'),'yyyy-MM-dd HH:mm:ss'))}"
} ]
}, {
"name" : "headerAttributeConfigs",
"value" : [ { } ]
}, {
"name" : "fieldAttributeConfigs",
"value" : [ {
"fieldToSet" : "/"
} ]
}, {
"name" : "stageOnRecordError",
"value" : "TO_ERROR"
}, {
"name" : "stageRequiredFields",
"value" : [ ]
}, {
"name" : "stageRecordPreconditions",
"value" : [ ]
} ],
"uiInfo" : {
"yPos" : 200,
"stageType" : "PROCESSOR",
"icon" : "expression.png",
"description" : "",
"label" : "yyyy-mm-dd Format to",
"xPos" : 499
},
"inputLanes" : [ "FieldTypeConverter_02OutputLane15884749055110" ],
"outputLanes" : [ "ExpressionEvaluator_02OutputLane15884750199420" ],
"eventLanes" : [ ],
"services" : [ ]
}, {
"instanceName" : "Trash_01",
"library" : "streamsets-datacollector-basic-lib",
"stageName" : "com_streamsets_pipeline_stage_destination_devnull_NullDTarget",
"stageVersion" : "1",
"configuration" : [ ],
"uiInfo" : {
"yPos" : 50,
"stageType" : "TARGET",
"icon" : "trash.png",
"description" : "",
"label" : "Trash",
"xPos" : 720
},
"inputLanes" : [ "ExpressionEvaluator_01OutputLane15884465930380" ],
"outputLanes" : [ ],
"eventLanes" : [ ],
"services" : [ ]
}, {
"instanceName" : "Trash_02",
"library" : "streamsets-datacollector-basic-lib",
"stageName" : "com_streamsets_pipeline_stage_destination_devnull_NullDTarget",
"stageVersion" : "1",
"configuration" : [ ],
"uiInfo" : {
"yPos" : 200,
"stageType" : "TARGET",
"icon" : "trash.png",
"description" : "",
"label" : "Trash",
"xPos" : 720
},
"inputLanes" : [ "ExpressionEvaluator_02OutputLane15884750199420" ],
"outputLanes" : [ ],
"eventLanes" : [ ],
"services" : [ ]
} ],
"errorStage" : {
"instanceName" : "Discard_ErrorStage",
"library" : "streamsets-datacollector-basic-lib",
"stageName" : "com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget",
"stageVersion" : "1",
"configuration" : [ ],
"uiInfo" : {
"yPos" : 50,
"stageType" : "TARGET",
"icon" : "",
"description" : "",
"label" : "Error Records - Discard",
"xPos" : 608
},
"inputLanes" : [ ],
"outputLanes" : [ ],
"eventLanes" : [ ],
"services" : [ ]
},
"info" : {
"pipelineId" : "FormattedMMddYYYYHHmmssdateto677507fb-d4d8-44bf-b01f-98350efbf5de",
"title" : "Formatted MM_dd_YYYY HH_mm_ss date to ...",
"description" : "",
"created" : 1588474826725,
"lastModified" : 1588475218324,
"creator" : "jeffrey.jennings",
"lastModifier" : "jeffrey.jennings",
"lastRev" : "0",
"uuid" : "ddf1c8ab-07dd-4318-bf7c-30fa2070ccc4",
"valid" : true,
"metadata" : {
"labels" : [ ]
},
"name" : "FormattedMMddYYYYHHmmssdateto677507fb-d4d8-44bf-b01f-98350efbf5de",
"sdcVersion" : "3.12.0",
"sdcId" : "90c52405-229d-11ea-bbf7-7b01a98a6646"
},
"metadata" : {
"labels" : [ ]
},
"statsAggregatorStage" : {
"instanceName" : "statsAggregatorStageInstance",
"library" : "streamsets-datacollector-basic-lib",
"stageName" : "com_streamsets_pipeline_stage_destination_devnull_StatsDpmDirectlyDTarget",
"stageVersion" : "1",
"configuration" : [ ],
"uiInfo" : {
"stageType" : "TARGET",
"label" : "Stats Aggregator -Write Directly to Control Hub - statistics are not aggregated across Data Collectors"
},
"inputLanes" : [ ],
"outputLanes" : [ ],
"eventLanes" : [ ],
"services" : [ ]
},
"startEventStages" : [ {
"instanceName" : "Discard_StartEventStage",
"library" : "streamsets-datacollector-basic-lib",
"stageName" : "com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget",
"stageVersion" : "1",
"configuration" : [ ],
"uiInfo" : {
"yPos" : 50,
"stageType" : "TARGET",
"icon" : "",
"description" : "",
"label" : "Start Event - Discard",
"xPos" : 358
},
"inputLanes" : [ ],
"outputLanes" : [ ],
"eventLanes" : [ ],
"services" : [ ]
} ],
"stopEventStages" : [ {
"instanceName" : "Discard_StopEventStage",
"library" : "streamsets-datacollector-basic-lib",
"stageName" : "com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget",
"stageVersion" : "1",
"configuration" : [ ],
"uiInfo" : {
"yPos" : 50,
"stageType" : "TARGET",
"icon" : "",
"description" : "",
"label" : "Stop Event - Discard",
"xPos" : 358
},
"inputLanes" : [ ],
"outputLanes" : [ ],
"eventLanes" : [ ],
"services" : [ ]
} ],
"testOriginStage" : {
"instanceName" : "com_streamsets_pipeline_stage_devtest_rawdata_RawDataDSource_TestOriginStage",
"library" : "streamsets-datacollector-dev-lib",
"stageName" : "com_streamsets_pipeline_stage_devtest_rawdata_RawDataDSource",
"stageVersion" : "3",
"configuration" : [ {
"name" : "rawData",
"value" : "{\n \"f1\": \"abc\",\n \"f2\": \"xyz\",\n \"f3\": \"lmn\"\n}"
}, {
"name" : "stopAfterFirstBatch",
"value" : false
}, {
"name" : "eventData",
"value" : null
}, {
"name" : "stageOnRecordError",
"value" : "TO_ERROR"
} ],
"uiInfo" : {
"stageType" : "SOURCE",
"label" : "Test Origin - Dev Raw Data Source"
},
"inputLanes" : [ ],
"outputLanes" : [ "com_streamsets_pipeline_stage_devtest_rawdata_RawDataDSource_TestOriginStageOutputLane1" ],
"eventLanes" : [ ],
"services" : [ {
"service" : "com.streamsets.pipeline.api.service.dataformats.DataFormatParserService",
"serviceVersion" : 1,
"configuration" : [ {
"name" : "displayFormats",
"value" : "DELIMITED,JSON,LOG,SDC_JSON,TEXT,XML"
}, {
"name" : "dataFormat",
"value" : "JSON"
}, {
"name" : "dataFormatConfig.compression",
"value" : "NONE"
}, {
"name" : "dataFormatConfig.filePatternInArchive",
"value" : "*"
}, {
"name" : "dataFormatConfig.charset",
"value" : "UTF-8"
}, {
"name" : "dataFormatConfig.removeCtrlChars",
"value" : false
}, {
"name" : "dataFormatConfig.textMaxLineLen",
"value" : 1024
}, {
"name" : "dataFormatConfig.useCustomDelimiter",
"value" : false
}, {
"name" : "dataFormatConfig.customDelimiter",
"value" : "\\r\\n"
}, {
"name" : "dataFormatConfig.includeCustomDelimiterInTheText",
"value" : false
}, {
"name" : "dataFormatConfig.jsonContent",
"value" : "MULTIPLE_OBJECTS"
}, {
"name" : "dataFormatConfig.jsonMaxObjectLen",
"value" : 4096
}, {
"name" : "dataFormatConfig.csvFileFormat",
"value" : "CSV"
}, {
"name" : "dataFormatConfig.csvHeader",
"value" : "NO_HEADER"
}, {
"name" : "dataFormatConfig.csvAllowExtraColumns",
"value" : false
}, {
"name" : "dataFormatConfig.csvExtraColumnPrefix",
"value" : "_extra_"
}, {
"name" : "dataFormatConfig.csvMaxObjectLen",
"value" : 1024
}, {
"name" : "dataFormatConfig.csvCustomDelimiter",
"value" : "|"
}, {
"name" : "dataFormatConfig.multiCharacterFieldDelimiter",
"value" : "||"
}, {
"name" : "dataFormatConfig.multiCharacterLineDelimiter",
"value" : "${str:unescapeJava('\\\\n')}"
}, {
"name" : "dataFormatConfig.csvCustomEscape",
"value" : "\\"
}, {
"name" : "dataFormatConfig.csvCustomQuote",
"value" : "\""
}, {
"name" : "dataFormatConfig.csvEnableComments",
"value" : false
}, {
"name" : "dataFormatConfig.csvCommentMarker",
"value" : "#"
}, {
"name" : "dataFormatConfig.csvIgnoreEmptyLines",
"value" : false
}, {
"name" : "dataFormatConfig.csvRecordType",
"value" : "LIST_MAP"
}, {
"name" : "dataFormatConfig.csvSkipStartLines",
"value" : 0
}, {
"name" : "dataFormatConfig.parseNull",
"value" : false
}, {
"name" : "dataFormatConfig.nullConstant",
"value" : "\\\\N"
}, {
"name" : "dataFormatConfig.xmlRecordElement",
"value" : null
}, {
"name" : "dataFormatConfig.includeFieldXpathAttributes",
"value" : false
}, {
"name" : "dataFormatConfig.xPathNamespaceContext",
"value" : [ ]
}, {
"name" : "dataFormatConfig.outputFieldAttributes",
"value" : false
}, {
"name" : "dataFormatConfig.xmlMaxObjectLen",
"value" : 4096
}, {
"name" : "dataFormatConfig.logMode",
"value" : "COMMON_LOG_FORMAT"
}, {
"name" : "dataFormatConfig.logMaxObjectLen",
"value" : 1024
}, {
"name" : "dataFormatConfig.retainOriginalLine",
"value" : false
}, {
"name" : "dataFormatConfig.customLogFormat",
"value" : "%h %l %u %t \"%r\" %>s %b"
}, {
"name" : "dataFormatConfig.regex",
"value" : "^(\\S+) (\\S+) (\\S+) \\[([\\w:/]+\\s[+\\-]\\d{4})\\] \"(\\S+) (\\S+) (\\S+)\" (\\d{3}) (\\d+)"
}, {
"name" : "dataFormatConfig.fieldPathsToGroupName",
"value" : [ {
"fieldPath" : "/",
"group" : 1
} ]
}, {
"name" : "dataFormatConfig.grokPatternDefinition",
"value" : null
}, {
"name" : "dataFormatConfig.grokPattern",
"value" : "%{COMMONAPACHELOG}"
}, {
"name" : "dataFormatConfig.onParseError",
"value" : "ERROR"
}, {
"name" : "dataFormatConfig.maxStackTraceLines",
"value" : 50
}, {
"name" : "dataFormatConfig.enableLog4jCustomLogFormat",
"value" : false
}, {
"name" : "dataFormatConfig.log4jCustomLogFormat",
"value" : "%r [%t] %-5p %c %x - %m%n"
}, {
"name" : "dataFormatConfig.avroSchemaSource",
"value" : null
}, {
"name" : "dataFormatConfig.avroSchema",
"value" : null
}, {
"name" : "dataFormatConfig.schemaRegistryUrls",
"value" : [ ]
}, {
"name" : "dataFormatConfig.basicAuth",
"value" : null
}, {
"name" : "dataFormatConfig.schemaLookupMode",
"value" : "SUBJECT"
}, {
"name" : "dataFormatConfig.subject",
"value" : null
}, {
"name" : "dataFormatConfig.schemaId",
"value" : null
}, {
"name" : "dataFormatConfig.avroSkipUnionIndex",
"value" : false
}, {
"name" : "dataFormatConfig.protoDescriptorFile",
"value" : null
}, {
"name" : "dataFormatConfig.messageType",
"value" : null
}, {
"name" : "dataFormatConfig.isDelimited",
"value" : false
}, {
"name" : "dataFormatConfig.binaryMaxObjectLen",
"value" : 1024
}, {
"name" : "dataFormatConfig.datagramMode",
"value" : "SYSLOG"
}, {
"name" : "dataFormatConfig.typesDbPath",
"value" : null
}, {
"name" : "dataFormatConfig.convertTime",
"value" : false
}, {
"name" : "dataFormatConfig.excludeInterval",
"value" : false
}, {
"name" : "dataFormatConfig.authFilePath",
"value" : null
}, {
"name" : "dataFormatConfig.netflowOutputValuesMode",
"value" : "RAW_AND_INTERPRETED"
}, {
"name" : "dataFormatConfig.maxTemplateCacheSize",
"value" : -1
}, {
"name" : "dataFormatConfig.templateCacheTimeoutMs",
"value" : -1
}, {
"name" : "dataFormatConfig.netflowOutputValuesModeDatagram",
"value" : "RAW_AND_INTERPRETED"
}, {
"name" : "dataFormatConfig.maxTemplateCacheSizeDatagram",
"value" : -1
}, {
"name" : "dataFormatConfig.templateCacheTimeoutMsDatagram",
"value" : -1
}, {
"name" : "dataFormatConfig.wholeFileMaxObjectLen",
"value" : 8192
}, {
"name" : "dataFormatConfig.rateLimit",
"value" : "-1"
}, {
"name" : "dataFormatConfig.verifyChecksum",
"value" : false
}, {
"name" : "dataFormatConfig.excelHeader",
"value" : null
}, {
"name" : "dataFormatConfig.excelSkipCellsWithNoHeader",
"value" : false
}, {
"name" : "dataFormatConfig.excelReadAllSheets",
"value" : false
}, {
"name" : "dataFormatConfig.excelSheetNames",
"value" : [ ]
} ]
} ]
},
"valid" : true,
"issues" : {
"pipelineIssues" : [ ],
"stageIssues" : { },
"issueCount" : 0
},
"previewable" : true
},
"pipelineRules" : {
"schemaVersion" : 3,
"version" : 2,
"metricsRuleDefinitions" : [ {
"id" : "badRecordsAlertID",
"alertText" : "High incidence of Error Records",
"metricId" : "pipeline.batchErrorRecords.counter",
"metricType" : "COUNTER",
"metricElement" : "COUNTER_COUNT",
"condition" : "${value() > 100}",
"sendEmail" : false,
"enabled" : false,
"timestamp" : 1588446219607,
"valid" : true
}, {
"id" : "stageErrorAlertID",
"alertText" : "High incidence of Stage Errors",
"metricId" : "pipeline.batchErrorMessages.counter",
"metricType" : "COUNTER",
"metricElement" : "COUNTER_COUNT",
"condition" : "${value() > 100}",
"sendEmail" : false,
"enabled" : false,
"timestamp" : 1588446219607,
"valid" : true
}, {
"id" : "idleGaugeID",
"alertText" : "Pipeline is Idle",
"metricId" : "RuntimeStatsGauge.gauge",
"metricType" : "GAUGE",
"metricElement" : "TIME_OF_LAST_RECEIVED_RECORD",
"condition" : "${time:now() - value() > 120000}",
"sendEmail" : false,
"enabled" : false,
"timestamp" : 1588446219607,
"valid" : true
}, {
"id" : "batchTimeAlertID",
"alertText" : "Batch taking more time to process",
"metricId" : "RuntimeStatsGauge.gauge",
"metricType" : "GAUGE",
"metricElement" : "CURRENT_BATCH_AGE",
"condition" : "${value() > 200}",
"sendEmail" : false,
"enabled" : false,
"timestamp" : 1588446219607,
"valid" : true
} ],
"dataRuleDefinitions" : [ ],
"driftRuleDefinitions" : [ ],
"uuid" : "cd209f35-1cef-419d-bf94-eebb2c034e9c",
"configuration" : [ {
"name" : "emailIDs",
"value" : [ ]
}, {
"name" : "webhookConfigs",
"value" : [ ]
} ],
"ruleIssues" : [ ],
"configIssues" : [ ]
},
"libraryDefinitions" : {
"schemaVersion" : "1",
"pipeline" : [ {
"configDefinitions" : [ {
"fieldName" : "executionMode",
"description" : "",
"max" : 9223372036854775807,
"label" : "Execution Mode",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : { },
"dependsOn" : "",
"group" : "",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "Standalone", "Cluster Batch", "Cluster Yarn Streaming", "Cluster Mesos Streaming", "Edge", "Cluster EMR Batch", "Batch", "Streaming" ],
"values" : [ "STANDALONE", "CLUSTER_BATCH", "CLUSTER_YARN_STREAMING", "CLUSTER_MESOS_STREAMING", "EDGE", "EMR_BATCH", "BATCH", "STREAMING" ],
"configDefinitions" : null,
"modelType" : "VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.datacollector.config.ExecutionModeChooserValues",
"filteringConfig" : ""
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 10,
"displayMode" : "BASIC",
"name" : "executionMode",
"type" : "MODEL",
"defaultValue" : "STANDALONE"
}, {
"fieldName" : "edgeHttpUrl",
"description" : "",
"max" : 9223372036854775807,
"label" : "Data Collector Edge URL",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"executionMode" : [ "EDGE" ]
},
"dependsOn" : "",
"group" : "",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 15,
"displayMode" : "BASIC",
"name" : "edgeHttpUrl",
"type" : "STRING",
"defaultValue" : "http://localhost:18633"
}, {
"fieldName" : "deliveryGuarantee",
"description" : "",
"max" : 9223372036854775807,
"label" : "Delivery Guarantee",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"executionMode" : [ "STANDALONE", "CLUSTER_BATCH", "EMR_BATCH", "CLUSTER_YARN_STREAMING", "EDGE", "CLUSTER_MESOS_STREAMING" ]
},
"dependsOn" : "",
"group" : "",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "At Least Once", "At Most Once" ],
"values" : [ "AT_LEAST_ONCE", "AT_MOST_ONCE" ],
"configDefinitions" : null,
"modelType" : "VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.datacollector.config.DeliveryGuaranteeChooserValues",
"filteringConfig" : ""
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 20,
"displayMode" : "BASIC",
"name" : "deliveryGuarantee",
"type" : "MODEL",
"defaultValue" : "AT_LEAST_ONCE"
}, {
"fieldName" : "testOriginStage",
"description" : "Stage used for testing in preview mode.",
"max" : 9223372036854775807,
"label" : "Test Origin",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : {
"executionMode" : [ "STANDALONE", "CLUSTER_BATCH", "EMR_BATCH", "CLUSTER_YARN_STREAMING", "EDGE", "CLUSTER_MESOS_STREAMING" ]
},
"dependsOn" : "",
"group" : "",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "Amazon S3 (Library: Amazon Web Services 1.11.123)", "Amazon SQS Consumer (Library: Amazon Web Services 1.11.123)", "Azure Data Lake Storage Gen1 (Library: Azure)", "Azure Data Lake Storage Gen2 (Library: Azure)", "Azure IoT/Event Hub Consumer (Library: Azure)", "CoAP Server (Library: Basic)", "Cron Scheduler (Library: Orchestrator)", "Dev Data Generator (Library: Dev (for development only))", "Dev Random Record Source (Library: Dev (for development only))", "Dev Raw Data Source (Library: Dev (for development only))", "Dev SDC RPC with Buffering (Library: Basic)", "Dev Snapshot Replaying (Library: Dev (for development only))", "Directory (Library: Basic)", "Elasticsearch (Library: Elasticsearch 5.2.0)", "File Tail (Library: Basic)", "Fragment Origin (Library: Basic)", "Google BigQuery (Library: Google Cloud)", "Google Cloud Storage (Library: Google Cloud)", "Google Pub Sub Subscriber (Library: Google Cloud)", "Groovy Scripting (Library: Groovy 2.4.11)", "gRPC Client (Library: Basic)", "Hadoop FS (Library: CDH 6.3.0)", "Hadoop FS (Library: EMR 5.14.0 - Hadoop 2.8.3)", "Hadoop FS (Library: CDH 5.14.0)", "Hadoop FS (Library: CDH 6.0.1)", "Hadoop FS (Library: HDP 3.1.0)", "Hadoop FS (Library: CDH 5.15.0)", "Hadoop FS (Library: CDH 6.1.1)", "Hadoop FS (Library: CDH 5.16.0)", "Hadoop FS (Library: CDH 6.2.0)", "Hadoop FS Standalone (Library: CDH 6.3.0)", "Hadoop FS Standalone (Library: CDH 5.14.0)", "Hadoop FS Standalone (Library: CDH 6.0.1)", "Hadoop FS Standalone (Library: Azure)", "Hadoop FS Standalone (Library: HDP 3.1.0)", "Hadoop FS Standalone (Library: CDH 5.15.0)", "Hadoop FS Standalone (Library: CDH 6.1.1)", "Hadoop FS Standalone (Library: CDH 5.16.0)", "Hadoop FS Standalone (Library: CDH 6.2.0)", "HTTP Client (Library: Basic)", "HTTP Server (Library: Basic)", "HTTP to Kafka (Library: Apache Kafka 1.1.0)", "HTTP to Kafka (Library: CDH 6.3.0)", "HTTP to Kafka (Library: Apache Kafka 2.0.0)", "HTTP to Kafka (Library: CDH 6.0.1)", "HTTP to Kafka (Library: HDP 3.1.0)", "HTTP to Kafka (Library: CDH Kafka 3.1.0)", "HTTP to Kafka (Library: Apache Kafka 1.0.0)", "HTTP to Kafka (Library: CDH 6.1.1)", "HTTP to Kafka (Library: CDH 6.2.0)", "JavaScript Scripting (Library: Basic)", "JDBC Multitable Consumer (Library: JDBC)", "JDBC Query Consumer (Library: JDBC)", "JMS Consumer (Library: JMS)", "Jython Scripting (Library: Jython 2.7.0)", "Kafka Consumer (Library: Apache Kafka 1.1.0)", "Kafka Consumer (Library: CDH 6.3.0)", "Kafka Consumer (Library: Apache Kafka 2.0.0)", "Kafka Consumer (Library: CDH Cluster Kafka with CDS Powered By Apache Spark 2.3.0 Release 4)", "Kafka Consumer (Library: CDH Cluster Kafka with CDS Powered By Apache Spark 2.2.0 Release 1)", "Kafka Consumer (Library: CDH Cluster Kafka with CDS Powered By Apache Spark 2.3.0 Release 2)", "Kafka Consumer (Library: CDH 6.0.1)", "Kafka Consumer (Library: HDP 3.1.0)", "Kafka Consumer (Library: CDH Kafka 3.1.0)", "Kafka Consumer (Library: Apache Kafka 1.0.0)", "Kafka Consumer (Library: CDH 6.1.1)", "Kafka Consumer (Library: CDH 6.2.0)", "Kafka Consumer (Library: CDH Cluster Kafka with CDS Powered By Apache Spark 2.3.0 Release 3)", "Kafka Multitopic Consumer (Library: Apache Kafka 1.1.0)", "Kafka Multitopic Consumer (Library: CDH 6.3.0)", "Kafka Multitopic Consumer (Library: Apache Kafka 2.0.0)", "Kafka Multitopic Consumer (Library: CDH 6.0.1)", "Kafka Multitopic Consumer (Library: HDP 3.1.0)", "Kafka Multitopic Consumer (Library: CDH Kafka 3.1.0)", "Kafka Multitopic Consumer (Library: Apache Kafka 1.0.0)", "Kafka Multitopic Consumer (Library: CDH 6.1.1)", "Kafka Multitopic Consumer (Library: CDH 6.2.0)", "Kinesis Consumer (Library: Amazon Kinesis)", "MongoDB (Library: MongoDB Java Driver 3.5.0)", "MongoDB Oplog (Library: MongoDB Java Driver 3.5.0)", "MQTT Subscriber (Library: Basic)", "MySQL Binary Log (Library: MySql BinLog)", "NiFi HTTP Server (Library: Basic)", "Omniture (Library: Omniture)", "OPC UA Client (Library: Basic)", "Oracle Bulkload (Library: Oracle Enterprise Library)", "Oracle CDC Client (Library: JDBC)", "PostgreSQL CDC Client (Library: JDBC)", "Pulsar Consumer (Library: Apache Pulsar 2.2.0)", "RabbitMQ Consumer (Library: RabbitMQ 3.5.6)", "Redis Consumer (Library: Redis 2.8.x, 3.0.x)", "REST Service (Library: Basic)", "Salesforce (Library: Salesforce Library)", "SDC RPC (Library: Basic)", "SDC RPC to Kafka (Library: Apache Kafka 1.1.0)", "SDC RPC to Kafka (Library: CDH 6.3.0)", "SDC RPC to Kafka (Library: Apache Kafka 2.0.0)", "SDC RPC to Kafka (Library: CDH 6.0.1)", "SDC RPC to Kafka (Library: HDP 3.1.0)", "SDC RPC to Kafka (Library: CDH Kafka 3.1.0)", "SDC RPC to Kafka (Library: Apache Kafka 1.0.0)", "SDC RPC to Kafka (Library: CDH 6.1.1)", "SDC RPC to Kafka (Library: CDH 6.2.0)", "Sensor Reader (Library: Dev (for development only))", "SFTP/FTP/FTPS Client (Library: Basic)", "SQL Server CDC Client (Library: JDBC)", "SQL Server Change Tracking Client (Library: JDBC)", "Start Pipeline (Library: Orchestrator)", "System Metrics (Library: Basic)", "TCP Server (Library: Basic)", "Teradata Consumer (Library: Teradata Enterprise Library)", "UDP Multithreaded Source (Library: Basic)", "UDP Source (Library: Basic)", "UDP to Kafka (Library: Apache Kafka 1.1.0)", "UDP to Kafka (Library: CDH 6.3.0)", "UDP to Kafka (Library: Apache Kafka 2.0.0)", "UDP to Kafka (Library: CDH 6.0.1)", "UDP to Kafka (Library: HDP 3.1.0)", "UDP to Kafka (Library: CDH Kafka 3.1.0)", "UDP to Kafka (Library: Apache Kafka 1.0.0)", "UDP to Kafka (Library: CDH 6.1.1)", "UDP to Kafka (Library: CDH 6.2.0)", "WebSocket Client (Library: Basic)", "WebSocket Server (Library: Basic)", "Windows Event Log (Library: Windows)" ],
"values" : [ "streamsets-datacollector-aws-lib::com_streamsets_pipeline_stage_origin_s3_AmazonS3DSource::11", "streamsets-datacollector-aws-lib::com_streamsets_pipeline_stage_origin_sqs_SqsDSource::3", "streamsets-datacollector-azure-lib::com_streamsets_pipeline_stage_origin_datalake_gen1_DataLakeDSource::1", "streamsets-datacollector-azure-lib::com_streamsets_pipeline_stage_origin_datalake_gen2_DataLakeGen2DSource::1", "streamsets-datacollector-azure-lib::com_streamsets_pipeline_stage_origin_eventhubs_EventHubConsumerDSource::1", "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_coapserver_CoapServerDPushSource::1", "streamsets-datacollector-orchestrator-lib::com_streamsets_pipeline_stage_origin_scheduler_SchedulerDPushSource::1", "streamsets-datacollector-dev-lib::com_streamsets_pipeline_stage_devtest_RandomDataGeneratorSource::5", "streamsets-datacollector-dev-lib::com_streamsets_pipeline_stage_devtest_RandomSource::1", "streamsets-datacollector-dev-lib::com_streamsets_pipeline_stage_devtest_rawdata_RawDataDSource::3", "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_sdcipcwithbuffer_SdcIpcWithDiskBufferDSource::3", "streamsets-datacollector-dev-lib::com_streamsets_pipeline_stage_devtest_replaying_SnapshotReplayDSource::1", "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_spooldir_SpoolDirDSource::10", "streamsets-datacollector-elasticsearch_5-lib::com_streamsets_pipeline_stage_origin_elasticsearch_ElasticsearchDSource::1", "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_logtail_FileTailDSource::4", "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_fragment_FragmentSource::1", "streamsets-datacollector-google-cloud-lib::com_streamsets_pipeline_stage_bigquery_origin_BigQueryDSource::2", "streamsets-datacollector-google-cloud-lib::com_streamsets_pipeline_stage_cloudstorage_origin_GoogleCloudStorageDSource::1", "streamsets-datacollector-google-cloud-lib::com_streamsets_pipeline_stage_pubsub_origin_PubSubDSource::1", "streamsets-datacollector-groovy_2_4-lib::com_streamsets_pipeline_stage_origin_groovy_GroovyDSource::1", "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_grpcclient_GrpcClientDSource::1", "streamsets-datacollector-cdh_6_3-lib::com_streamsets_pipeline_stage_origin_hdfs_cluster_ClusterHdfsDSource::6", "streamsets-datacollector-emr_hadoop_2_8_3-lib::com_streamsets_pipeline_stage_origin_hdfs_cluster_ClusterHdfsDSource::6", "streamsets-datacollector-cdh_5_14-lib::com_streamsets_pipeline_stage_origin_hdfs_cluster_ClusterHdfsDSource::6", "streamsets-datacollector-cdh_6_0-lib::com_streamsets_pipeline_stage_origin_hdfs_cluster_ClusterHdfsDSource::6", "streamsets-datacollector-hdp_3_1-lib::com_streamsets_pipeline_stage_origin_hdfs_cluster_ClusterHdfsDSource::6", "streamsets-datacollector-cdh_5_15-lib::com_streamsets_pipeline_stage_origin_hdfs_cluster_ClusterHdfsDSource::6", "streamsets-datacollector-cdh_6_1-lib::com_streamsets_pipeline_stage_origin_hdfs_cluster_ClusterHdfsDSource::6", "streamsets-datacollector-cdh_5_16-lib::com_streamsets_pipeline_stage_origin_hdfs_cluster_ClusterHdfsDSource::6", "streamsets-datacollector-cdh_6_2-lib::com_streamsets_pipeline_stage_origin_hdfs_cluster_ClusterHdfsDSource::6", "streamsets-datacollector-cdh_6_3-lib::com_streamsets_pipeline_stage_origin_hdfs_HdfsDSource::1", "streamsets-datacollector-cdh_5_14-lib::com_streamsets_pipeline_stage_origin_hdfs_HdfsDSource::1", "streamsets-datacollector-cdh_6_0-lib::com_streamsets_pipeline_stage_origin_hdfs_HdfsDSource::1", "streamsets-datacollector-azure-lib::com_streamsets_pipeline_stage_origin_hdfs_HdfsDSource::1", "streamsets-datacollector-hdp_3_1-lib::com_streamsets_pipeline_stage_origin_hdfs_HdfsDSource::1", "streamsets-datacollector-cdh_5_15-lib::com_streamsets_pipeline_stage_origin_hdfs_HdfsDSource::1", "streamsets-datacollector-cdh_6_1-lib::com_streamsets_pipeline_stage_origin_hdfs_HdfsDSource::1", "streamsets-datacollector-cdh_5_16-lib::com_streamsets_pipeline_stage_origin_hdfs_HdfsDSource::1", "streamsets-datacollector-cdh_6_2-lib::com_streamsets_pipeline_stage_origin_hdfs_HdfsDSource::1", "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_http_HttpClientDSource::14", "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_httpserver_HttpServerDPushSource::10", "streamsets-datacollector-apache-kafka_1_1-lib::com_streamsets_pipeline_stage_origin_httptokafka_HttpToKafkaDSource::2", "streamsets-datacollector-cdh_6_3-lib::com_streamsets_pipeline_stage_origin_httptokafka_HttpToKafkaDSource::2", "streamsets-datacollector-apache-kafka_2_0-lib::com_streamsets_pipeline_stage_origin_httptokafka_HttpToKafkaDSource::2", "streamsets-datacollector-cdh_6_0-lib::com_streamsets_pipeline_stage_origin_httptokafka_HttpToKafkaDSource::2", "streamsets-datacollector-hdp_3_1-lib::com_streamsets_pipeline_stage_origin_httptokafka_HttpToKafkaDSource::2", "streamsets-datacollector-cdh_kafka_3_1-lib::com_streamsets_pipeline_stage_origin_httptokafka_HttpToKafkaDSource::2", "streamsets-datacollector-apache-kafka_1_0-lib::com_streamsets_pipeline_stage_origin_httptokafka_HttpToKafkaDSource::2", "streamsets-datacollector-cdh_6_1-lib::com_streamsets_pipeline_stage_origin_httptokafka_HttpToKafkaDSource::2", "streamsets-datacollector-cdh_6_2-lib::com_streamsets_pipeline_stage_origin_httptokafka_HttpToKafkaDSource::2", "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_javascript_JavascriptDSource::1", "streamsets-datacollector-jdbc-lib::com_streamsets_pipeline_stage_origin_jdbc_table_TableJdbcDSource::5", "streamsets-datacollector-jdbc-lib::com_streamsets_pipeline_stage_origin_jdbc_JdbcDSource::10", "streamsets-datacollector-jms-lib::com_streamsets_pipeline_stage_origin_jms_JmsDSource::7", "streamsets-datacollector-jython_2_7-lib::com_streamsets_pipeline_stage_origin_jython_JythonDSource::1", "streamsets-datacollector-apache-kafka_1_1-lib::com_streamsets_pipeline_stage_origin_kafka_KafkaDSource::9", "streamsets-datacollector-cdh_6_3-lib::com_streamsets_pipeline_stage_origin_kafka_KafkaDSource::9", "streamsets-datacollector-apache-kafka_2_0-lib::com_streamsets_pipeline_stage_origin_kafka_KafkaDSource::9", "streamsets-datacollector-cdh-spark_2_3_r4-lib::com_streamsets_pipeline_stage_origin_kafka_KafkaDSource::9", "streamsets-datacollector-cdh-spark_2_2-lib::com_streamsets_pipeline_stage_origin_kafka_KafkaDSource::9", "streamsets-datacollector-cdh-spark_2_3-lib::com_streamsets_pipeline_stage_origin_kafka_KafkaDSource::9", "streamsets-datacollector-cdh_6_0-lib::com_streamsets_pipeline_stage_origin_kafka_KafkaDSource::9", "streamsets-datacollector-hdp_3_1-lib::com_streamsets_pipeline_stage_origin_kafka_KafkaDSource::9", "streamsets-datacollector-cdh_kafka_3_1-lib::com_streamsets_pipeline_stage_origin_kafka_KafkaDSource::9", "streamsets-datacollector-apache-kafka_1_0-lib::com_streamsets_pipeline_stage_origin_kafka_KafkaDSource::9", "streamsets-datacollector-cdh_6_1-lib::com_streamsets_pipeline_stage_origin_kafka_KafkaDSource::9", "streamsets-datacollector-cdh_6_2-lib::com_streamsets_pipeline_stage_origin_kafka_KafkaDSource::9", "streamsets-datacollector-cdh-spark_2_3_r3-lib::com_streamsets_pipeline_stage_origin_kafka_KafkaDSource::9", "streamsets-datacollector-apache-kafka_1_1-lib::com_streamsets_pipeline_stage_origin_multikafka_MultiKafkaDSource::4", "streamsets-datacollector-cdh_6_3-lib::com_streamsets_pipeline_stage_origin_multikafka_MultiKafkaDSource::4", "streamsets-datacollector-apache-kafka_2_0-lib::com_streamsets_pipeline_stage_origin_multikafka_MultiKafkaDSource::4", "streamsets-datacollector-cdh_6_0-lib::com_streamsets_pipeline_stage_origin_multikafka_MultiKafkaDSource::4", "streamsets-datacollector-hdp_3_1-lib::com_streamsets_pipeline_stage_origin_multikafka_MultiKafkaDSource::4", "streamsets-datacollector-cdh_kafka_3_1-lib::com_streamsets_pipeline_stage_origin_multikafka_MultiKafkaDSource::4", "streamsets-datacollector-apache-kafka_1_0-lib::com_streamsets_pipeline_stage_origin_multikafka_MultiKafkaDSource::4", "streamsets-datacollector-cdh_6_1-lib::com_streamsets_pipeline_stage_origin_multikafka_MultiKafkaDSource::4", "streamsets-datacollector-cdh_6_2-lib::com_streamsets_pipeline_stage_origin_multikafka_MultiKafkaDSource::4", "streamsets-datacollector-kinesis-lib::com_streamsets_pipeline_stage_origin_kinesis_KinesisDSource::6", "streamsets-datacollector-mongodb_3-lib::com_streamsets_pipeline_stage_origin_mongodb_MongoDBDSource::3", "streamsets-datacollector-mongodb_3-lib::com_streamsets_pipeline_stage_origin_mongodb_oplog_MongoDBOplogDSource::1", "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_mqtt_MqttClientDSource::3", "streamsets-datacollector-mysql-binlog-lib::com_streamsets_pipeline_stage_origin_mysql_MysqlDSource::2", "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_httpserver_nifi_NiFiHttpServerDPushSource::1", "streamsets-datacollector-omniture-lib::com_streamsets_pipeline_stage_origin_omniture_OmnitureDSource::2", "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_opcua_OpcUaClientDSource::3", "streamsets-datacollector-oracle-lib::com_streamsets_pipeline_stage_origin_oracle_OracleDSource::1", "streamsets-datacollector-jdbc-lib::com_streamsets_pipeline_stage_origin_jdbc_cdc_oracle_OracleCDCDSource::10", "streamsets-datacollector-jdbc-lib::com_streamsets_pipeline_stage_origin_jdbc_cdc_postgres_PostgresCDCDSource::1", "streamsets-datacollector-apache-pulsar_2-lib::com_streamsets_pipeline_stage_origin_pulsar_PulsarDSource::1", "streamsets-datacollector-rabbitmq-lib::com_streamsets_pipeline_stage_origin_rabbitmq_RabbitDSource::4", "streamsets-datacollector-redis-lib::com_streamsets_pipeline_stage_origin_redis_RedisDSource::3", "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_restservice_RestServiceDPushSource::3", "streamsets-datacollector-salesforce-lib::com_streamsets_pipeline_stage_origin_salesforce_ForceDSource::2", "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_sdcipc_SdcIpcDSource::2", "streamsets-datacollector-apache-kafka_1_1-lib::com_streamsets_pipeline_stage_origin_ipctokafka_SdcIpcToKafkaDSource::4", "streamsets-datacollector-cdh_6_3-lib::com_streamsets_pipeline_stage_origin_ipctokafka_SdcIpcToKafkaDSource::4", "streamsets-datacollector-apache-kafka_2_0-lib::com_streamsets_pipeline_stage_origin_ipctokafka_SdcIpcToKafkaDSource::4", "streamsets-datacollector-cdh_6_0-lib::com_streamsets_pipeline_stage_origin_ipctokafka_SdcIpcToKafkaDSource::4", "streamsets-datacollector-hdp_3_1-lib::com_streamsets_pipeline_stage_origin_ipctokafka_SdcIpcToKafkaDSource::4", "streamsets-datacollector-cdh_kafka_3_1-lib::com_streamsets_pipeline_stage_origin_ipctokafka_SdcIpcToKafkaDSource::4", "streamsets-datacollector-apache-kafka_1_0-lib::com_streamsets_pipeline_stage_origin_ipctokafka_SdcIpcToKafkaDSource::4", "streamsets-datacollector-cdh_6_1-lib::com_streamsets_pipeline_stage_origin_ipctokafka_SdcIpcToKafkaDSource::4", "streamsets-datacollector-cdh_6_2-lib::com_streamsets_pipeline_stage_origin_ipctokafka_SdcIpcToKafkaDSource::4", "streamsets-datacollector-dev-lib::com_streamsets_pipeline_stage_origin_sensorreader_SensorReaderDSource::1", "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_remote_RemoteDownloadDSource::4", "streamsets-datacollector-jdbc-lib::com_streamsets_pipeline_stage_origin_jdbc_cdc_sqlserver_SQLServerCDCDSource::5", "streamsets-datacollector-jdbc-lib::com_streamsets_pipeline_stage_origin_jdbc_CT_sqlserver_SQLServerCTDSource::2", "streamsets-datacollector-orchestrator-lib::com_streamsets_pipeline_stage_origin_startPipeline_StartPipelineDSource::1", "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_systemmetrics_SystemMetricsDSource::2", "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_tcp_TCPServerDSource::4", "streamsets-datacollector-teradata-lib::com_streamsets_pipeline_stage_origin_teradata_TeradataTableJdbcDSource::1", "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_udp_MultithreadedUDPDSource::1", "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_udp_UDPDSource::4", "streamsets-datacollector-apache-kafka_1_1-lib::com_streamsets_pipeline_stage_origin_udptokafka_UDPToKafkaDSource::3", "streamsets-datacollector-cdh_6_3-lib::com_streamsets_pipeline_stage_origin_udptokafka_UDPToKafkaDSource::3", "streamsets-datacollector-apache-kafka_2_0-lib::com_streamsets_pipeline_stage_origin_udptokafka_UDPToKafkaDSource::3", "streamsets-datacollector-cdh_6_0-lib::com_streamsets_pipeline_stage_origin_udptokafka_UDPToKafkaDSource::3", "streamsets-datacollector-hdp_3_1-lib::com_streamsets_pipeline_stage_origin_udptokafka_UDPToKafkaDSource::3", "streamsets-datacollector-cdh_kafka_3_1-lib::com_streamsets_pipeline_stage_origin_udptokafka_UDPToKafkaDSource::3", "streamsets-datacollector-apache-kafka_1_0-lib::com_streamsets_pipeline_stage_origin_udptokafka_UDPToKafkaDSource::3", "streamsets-datacollector-cdh_6_1-lib::com_streamsets_pipeline_stage_origin_udptokafka_UDPToKafkaDSource::3", "streamsets-datacollector-cdh_6_2-lib::com_streamsets_pipeline_stage_origin_udptokafka_UDPToKafkaDSource::3", "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_websocket_WebSocketClientDSource::4", "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_origin_websocketserver_WebSocketServerDPushSource::12", "streamsets-datacollector-windows-lib::com_streamsets_pipeline_stage_origin_windows_WindowsEventLogDSource::3" ],
"configDefinitions" : null,
"modelType" : "VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.datacollector.config.PipelineTestStageChooserValues",
"filteringConfig" : ""
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 21,
"displayMode" : "BASIC",
"name" : "testOriginStage",
"type" : "MODEL",
"defaultValue" : "streamsets-datacollector-dev-lib::com_streamsets_pipeline_stage_devtest_rawdata_RawDataDSource::3"
}, {
"fieldName" : "startEventStage",
"description" : "Stage that should handle pipeline start event.",
"max" : 9223372036854775807,
"label" : "Start Event",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : {
"executionMode" : [ "STANDALONE" ]
},
"dependsOn" : "",
"group" : "",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "ADLS Gen1 File Metadata (Library: Azure)", "ADLS Gen2 File Metadata (Library: Azure)", "Amazon S3 (Library: Amazon Web Services 1.11.123)", "Databricks Job Launcher (Library: Basic)", "Discard (Library: Basic)", "Email (Library: Basic)", "HDFS File Metadata (Library: CDH 6.3.0)", "HDFS File Metadata (Library: CDH 5.14.0)", "HDFS File Metadata (Library: CDH 6.0.1)", "HDFS File Metadata (Library: Azure)", "HDFS File Metadata (Library: HDP 3.1.0)", "HDFS File Metadata (Library: CDH 5.15.0)", "HDFS File Metadata (Library: CDH 6.1.1)", "HDFS File Metadata (Library: CDH 5.16.0)", "HDFS File Metadata (Library: CDH 6.2.0)", "Hive Query (Library: CDH 6.3.0)", "Hive Query (Library: CDH 5.14.0)", "Hive Query (Library: CDH 6.0.1)", "Hive Query (Library: HDP 3.1.0)", "Hive Query (Library: CDH 5.15.0)", "Hive Query (Library: CDH 6.1.1)", "Hive Query (Library: CDH 5.16.0)", "Hive Query (Library: CDH 6.2.0)", "JDBC Query (Library: JDBC)", "MapReduce (Library: CDH 6.3.0)", "MapReduce (Library: CDH 5.14.0)", "MapReduce (Library: CDH 6.0.1)", "MapReduce (Library: HDP 3.1.0)", "MapReduce (Library: CDH 5.15.0)", "MapReduce (Library: CDH 6.1.1)", "MapReduce (Library: CDH 5.16.0)", "MapReduce (Library: CDH 6.2.0)", "Shell (Library: Basic)", "Spark (Library: CDH 6.3.0)", "Spark (Library: CDH 5.14.0)", "Spark (Library: CDH 6.0.1)", "Spark (Library: CDH Spark 2.1.0 Release 1)", "Spark (Library: HDP 3.1.0)", "Spark (Library: CDH 5.15.0)", "Spark (Library: CDH 6.1.1)", "Spark (Library: CDH 5.16.0)", "Spark (Library: CDH 6.2.0)", "Write to Another Pipeline (Library: Basic)" ],
"values" : [ "streamsets-datacollector-azure-lib::com_streamsets_pipeline_stage_metadata_gen1_DataLakeMetadataDExecutor::1", "streamsets-datacollector-azure-lib::com_streamsets_pipeline_stage_metadata_gen2_DataLakeGen2MetadataDExecutor::1", "streamsets-datacollector-aws-lib::com_streamsets_pipeline_stage_executor_s3_AmazonS3DExecutor::1", "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_executor_databricks_DatabricksJobLauncherDExecutor::1", "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1", "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_executor_emailexecutor_EmailDExecutor::1", "streamsets-datacollector-cdh_6_3-lib::com_streamsets_pipeline_stage_destination_hdfs_metadataexecutor_HdfsMetadataDExecutor::1", "streamsets-datacollector-cdh_5_14-lib::com_streamsets_pipeline_stage_destination_hdfs_metadataexecutor_HdfsMetadataDExecutor::1", "streamsets-datacollector-cdh_6_0-lib::com_streamsets_pipeline_stage_destination_hdfs_metadataexecutor_HdfsMetadataDExecutor::1", "streamsets-datacollector-azure-lib::com_streamsets_pipeline_stage_destination_hdfs_metadataexecutor_HdfsMetadataDExecutor::1", "streamsets-datacollector-hdp_3_1-lib::com_streamsets_pipeline_stage_destination_hdfs_metadataexecutor_HdfsMetadataDExecutor::1", "streamsets-datacollector-cdh_5_15-lib::com_streamsets_pipeline_stage_destination_hdfs_metadataexecutor_HdfsMetadataDExecutor::1", "streamsets-datacollector-cdh_6_1-lib::com_streamsets_pipeline_stage_destination_hdfs_metadataexecutor_HdfsMetadataDExecutor::1", "streamsets-datacollector-cdh_5_16-lib::com_streamsets_pipeline_stage_destination_hdfs_metadataexecutor_HdfsMetadataDExecutor::1", "streamsets-datacollector-cdh_6_2-lib::com_streamsets_pipeline_stage_destination_hdfs_metadataexecutor_HdfsMetadataDExecutor::1", "streamsets-datacollector-cdh_6_3-lib::com_streamsets_pipeline_stage_destination_hive_queryexecutor_HiveQueryDExecutor::2", "streamsets-datacollector-cdh_5_14-lib::com_streamsets_pipeline_stage_destination_hive_queryexecutor_HiveQueryDExecutor::2", "streamsets-datacollector-cdh_6_0-lib::com_streamsets_pipeline_stage_destination_hive_queryexecutor_HiveQueryDExecutor::2", "streamsets-datacollector-hdp_3_1-lib::com_streamsets_pipeline_stage_destination_hive_queryexecutor_HiveQueryDExecutor::2", "streamsets-datacollector-cdh_5_15-lib::com_streamsets_pipeline_stage_destination_hive_queryexecutor_HiveQueryDExecutor::2", "streamsets-datacollector-cdh_6_1-lib::com_streamsets_pipeline_stage_destination_hive_queryexecutor_HiveQueryDExecutor::2", "streamsets-datacollector-cdh_5_16-lib::com_streamsets_pipeline_stage_destination_hive_queryexecutor_HiveQueryDExecutor::2", "streamsets-datacollector-cdh_6_2-lib::com_streamsets_pipeline_stage_destination_hive_queryexecutor_HiveQueryDExecutor::2", "streamsets-datacollector-jdbc-lib::com_streamsets_pipeline_stage_executor_jdbc_JdbcQueryDExecutor::3", "streamsets-datacollector-cdh_6_3-lib::com_streamsets_pipeline_stage_destination_mapreduce_MapReduceDExecutor::3", "streamsets-datacollector-cdh_5_14-lib::com_streamsets_pipeline_stage_destination_mapreduce_MapReduceDExecutor::3", "streamsets-datacollector-cdh_6_0-lib::com_streamsets_pipeline_stage_destination_mapreduce_MapReduceDExecutor::3", "streamsets-datacollector-hdp_3_1-lib::com_streamsets_pipeline_stage_destination_mapreduce_MapReduceDExecutor::3", "streamsets-datacollector-cdh_5_15-lib::com_streamsets_pipeline_stage_destination_mapreduce_MapReduceDExecutor::3", "streamsets-datacollector-cdh_6_1-lib::com_streamsets_pipeline_stage_destination_mapreduce_MapReduceDExecutor::3", "streamsets-datacollector-cdh_5_16-lib::com_streamsets_pipeline_stage_destination_mapreduce_MapReduceDExecutor::3", "streamsets-datacollector-cdh_6_2-lib::com_streamsets_pipeline_stage_destination_mapreduce_MapReduceDExecutor::3", "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_executor_shell_ShellDExecutor::1", "streamsets-datacollector-cdh_6_3-lib::com_streamsets_datacollector_pipeline_executor_spark_SparkDExecutor::2", "streamsets-datacollector-cdh_5_14-lib::com_streamsets_datacollector_pipeline_executor_spark_SparkDExecutor::2", "streamsets-datacollector-cdh_6_0-lib::com_streamsets_datacollector_pipeline_executor_spark_SparkDExecutor::2", "streamsets-datacollector-cdh_spark_2_1_r1-lib::com_streamsets_datacollector_pipeline_executor_spark_SparkDExecutor::2", "streamsets-datacollector-hdp_3_1-lib::com_streamsets_datacollector_pipeline_executor_spark_SparkDExecutor::2", "streamsets-datacollector-cdh_5_15-lib::com_streamsets_datacollector_pipeline_executor_spark_SparkDExecutor::2", "streamsets-datacollector-cdh_6_1-lib::com_streamsets_datacollector_pipeline_executor_spark_SparkDExecutor::2", "streamsets-datacollector-cdh_5_16-lib::com_streamsets_datacollector_pipeline_executor_spark_SparkDExecutor::2", "streamsets-datacollector-cdh_6_2-lib::com_streamsets_datacollector_pipeline_executor_spark_SparkDExecutor::2", "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_sdcipc_ToErrorSdcIpcDTarget::2" ],
"configDefinitions" : null,
"modelType" : "VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.datacollector.config.PipelineLifecycleStageChooserValues",
"filteringConfig" : ""
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 23,
"displayMode" : "BASIC",
"name" : "startEventStage",
"type" : "MODEL",
"defaultValue" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1"
}, {
"fieldName" : "stopEventStage",
"description" : "Stage that should handle pipeline stop event.",
"max" : 9223372036854775807,
"label" : "Stop Event",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : {
"executionMode" : [ "STANDALONE" ]
},
"dependsOn" : "",
"group" : "",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "ADLS Gen1 File Metadata (Library: Azure)", "ADLS Gen2 File Metadata (Library: Azure)", "Amazon S3 (Library: Amazon Web Services 1.11.123)", "Databricks Job Launcher (Library: Basic)", "Discard (Library: Basic)", "Email (Library: Basic)", "HDFS File Metadata (Library: CDH 6.3.0)", "HDFS File Metadata (Library: CDH 5.14.0)", "HDFS File Metadata (Library: CDH 6.0.1)", "HDFS File Metadata (Library: Azure)", "HDFS File Metadata (Library: HDP 3.1.0)", "HDFS File Metadata (Library: CDH 5.15.0)", "HDFS File Metadata (Library: CDH 6.1.1)", "HDFS File Metadata (Library: CDH 5.16.0)", "HDFS File Metadata (Library: CDH 6.2.0)", "Hive Query (Library: CDH 6.3.0)", "Hive Query (Library: CDH 5.14.0)", "Hive Query (Library: CDH 6.0.1)", "Hive Query (Library: HDP 3.1.0)", "Hive Query (Library: CDH 5.15.0)", "Hive Query (Library: CDH 6.1.1)", "Hive Query (Library: CDH 5.16.0)", "Hive Query (Library: CDH 6.2.0)", "JDBC Query (Library: JDBC)", "MapReduce (Library: CDH 6.3.0)", "MapReduce (Library: CDH 5.14.0)", "MapReduce (Library: CDH 6.0.1)", "MapReduce (Library: HDP 3.1.0)", "MapReduce (Library: CDH 5.15.0)", "MapReduce (Library: CDH 6.1.1)", "MapReduce (Library: CDH 5.16.0)", "MapReduce (Library: CDH 6.2.0)", "Shell (Library: Basic)", "Spark (Library: CDH 6.3.0)", "Spark (Library: CDH 5.14.0)", "Spark (Library: CDH 6.0.1)", "Spark (Library: CDH Spark 2.1.0 Release 1)", "Spark (Library: HDP 3.1.0)", "Spark (Library: CDH 5.15.0)", "Spark (Library: CDH 6.1.1)", "Spark (Library: CDH 5.16.0)", "Spark (Library: CDH 6.2.0)", "Write to Another Pipeline (Library: Basic)" ],
"values" : [ "streamsets-datacollector-azure-lib::com_streamsets_pipeline_stage_metadata_gen1_DataLakeMetadataDExecutor::1", "streamsets-datacollector-azure-lib::com_streamsets_pipeline_stage_metadata_gen2_DataLakeGen2MetadataDExecutor::1", "streamsets-datacollector-aws-lib::com_streamsets_pipeline_stage_executor_s3_AmazonS3DExecutor::1", "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_executor_databricks_DatabricksJobLauncherDExecutor::1", "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1", "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_executor_emailexecutor_EmailDExecutor::1", "streamsets-datacollector-cdh_6_3-lib::com_streamsets_pipeline_stage_destination_hdfs_metadataexecutor_HdfsMetadataDExecutor::1", "streamsets-datacollector-cdh_5_14-lib::com_streamsets_pipeline_stage_destination_hdfs_metadataexecutor_HdfsMetadataDExecutor::1", "streamsets-datacollector-cdh_6_0-lib::com_streamsets_pipeline_stage_destination_hdfs_metadataexecutor_HdfsMetadataDExecutor::1", "streamsets-datacollector-azure-lib::com_streamsets_pipeline_stage_destination_hdfs_metadataexecutor_HdfsMetadataDExecutor::1", "streamsets-datacollector-hdp_3_1-lib::com_streamsets_pipeline_stage_destination_hdfs_metadataexecutor_HdfsMetadataDExecutor::1", "streamsets-datacollector-cdh_5_15-lib::com_streamsets_pipeline_stage_destination_hdfs_metadataexecutor_HdfsMetadataDExecutor::1", "streamsets-datacollector-cdh_6_1-lib::com_streamsets_pipeline_stage_destination_hdfs_metadataexecutor_HdfsMetadataDExecutor::1", "streamsets-datacollector-cdh_5_16-lib::com_streamsets_pipeline_stage_destination_hdfs_metadataexecutor_HdfsMetadataDExecutor::1", "streamsets-datacollector-cdh_6_2-lib::com_streamsets_pipeline_stage_destination_hdfs_metadataexecutor_HdfsMetadataDExecutor::1", "streamsets-datacollector-cdh_6_3-lib::com_streamsets_pipeline_stage_destination_hive_queryexecutor_HiveQueryDExecutor::2", "streamsets-datacollector-cdh_5_14-lib::com_streamsets_pipeline_stage_destination_hive_queryexecutor_HiveQueryDExecutor::2", "streamsets-datacollector-cdh_6_0-lib::com_streamsets_pipeline_stage_destination_hive_queryexecutor_HiveQueryDExecutor::2", "streamsets-datacollector-hdp_3_1-lib::com_streamsets_pipeline_stage_destination_hive_queryexecutor_HiveQueryDExecutor::2", "streamsets-datacollector-cdh_5_15-lib::com_streamsets_pipeline_stage_destination_hive_queryexecutor_HiveQueryDExecutor::2", "streamsets-datacollector-cdh_6_1-lib::com_streamsets_pipeline_stage_destination_hive_queryexecutor_HiveQueryDExecutor::2", "streamsets-datacollector-cdh_5_16-lib::com_streamsets_pipeline_stage_destination_hive_queryexecutor_HiveQueryDExecutor::2", "streamsets-datacollector-cdh_6_2-lib::com_streamsets_pipeline_stage_destination_hive_queryexecutor_HiveQueryDExecutor::2", "streamsets-datacollector-jdbc-lib::com_streamsets_pipeline_stage_executor_jdbc_JdbcQueryDExecutor::3", "streamsets-datacollector-cdh_6_3-lib::com_streamsets_pipeline_stage_destination_mapreduce_MapReduceDExecutor::3", "streamsets-datacollector-cdh_5_14-lib::com_streamsets_pipeline_stage_destination_mapreduce_MapReduceDExecutor::3", "streamsets-datacollector-cdh_6_0-lib::com_streamsets_pipeline_stage_destination_mapreduce_MapReduceDExecutor::3", "streamsets-datacollector-hdp_3_1-lib::com_streamsets_pipeline_stage_destination_mapreduce_MapReduceDExecutor::3", "streamsets-datacollector-cdh_5_15-lib::com_streamsets_pipeline_stage_destination_mapreduce_MapReduceDExecutor::3", "streamsets-datacollector-cdh_6_1-lib::com_streamsets_pipeline_stage_destination_mapreduce_MapReduceDExecutor::3", "streamsets-datacollector-cdh_5_16-lib::com_streamsets_pipeline_stage_destination_mapreduce_MapReduceDExecutor::3", "streamsets-datacollector-cdh_6_2-lib::com_streamsets_pipeline_stage_destination_mapreduce_MapReduceDExecutor::3", "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_executor_shell_ShellDExecutor::1", "streamsets-datacollector-cdh_6_3-lib::com_streamsets_datacollector_pipeline_executor_spark_SparkDExecutor::2", "streamsets-datacollector-cdh_5_14-lib::com_streamsets_datacollector_pipeline_executor_spark_SparkDExecutor::2", "streamsets-datacollector-cdh_6_0-lib::com_streamsets_datacollector_pipeline_executor_spark_SparkDExecutor::2", "streamsets-datacollector-cdh_spark_2_1_r1-lib::com_streamsets_datacollector_pipeline_executor_spark_SparkDExecutor::2", "streamsets-datacollector-hdp_3_1-lib::com_streamsets_datacollector_pipeline_executor_spark_SparkDExecutor::2", "streamsets-datacollector-cdh_5_15-lib::com_streamsets_datacollector_pipeline_executor_spark_SparkDExecutor::2", "streamsets-datacollector-cdh_6_1-lib::com_streamsets_datacollector_pipeline_executor_spark_SparkDExecutor::2", "streamsets-datacollector-cdh_5_16-lib::com_streamsets_datacollector_pipeline_executor_spark_SparkDExecutor::2", "streamsets-datacollector-cdh_6_2-lib::com_streamsets_datacollector_pipeline_executor_spark_SparkDExecutor::2", "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_sdcipc_ToErrorSdcIpcDTarget::2" ],
"configDefinitions" : null,
"modelType" : "VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.datacollector.config.PipelineLifecycleStageChooserValues",
"filteringConfig" : ""
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 26,
"displayMode" : "BASIC",
"name" : "stopEventStage",
"type" : "MODEL",
"defaultValue" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1"
}, {
"fieldName" : "shouldRetry",
"description" : "",
"max" : 9223372036854775807,
"label" : "Retry Pipeline on Error",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"executionMode" : [ "STANDALONE", "CLUSTER_BATCH", "CLUSTER_YARN_STREAMING", "CLUSTER_MESOS_STREAMING" ]
},
"dependsOn" : "",
"group" : "",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 30,
"displayMode" : "BASIC",
"name" : "shouldRetry",
"type" : "BOOLEAN",
"defaultValue" : true
}, {
"fieldName" : "triggerInterval",
"description" : "Time interval between generation of batches",
"max" : 9223372036854775807,
"label" : "Trigger Interval (millis)",
"required" : false,
"min" : 1,
"dependsOnMap" : {
"executionMode" : [ "STREAMING" ]
},
"dependsOn" : "",
"group" : "",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 35,
"displayMode" : "BASIC",
"name" : "triggerInterval",
"type" : "NUMBER",
"defaultValue" : 2000
}, {
"fieldName" : "retryAttempts",
"description" : "Max no of retries. To retry indefinitely, use -1. The wait time between retries starts at 15 seconds and doubles until reaching 5 minutes.",
"max" : 9223372036854775807,
"label" : "Retry Attempts",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : {
"shouldRetry" : [ true ],
"executionMode" : [ "STANDALONE", "CLUSTER_BATCH", "CLUSTER_YARN_STREAMING", "CLUSTER_MESOS_STREAMING" ]
},
"dependsOn" : "",
"group" : "",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 30,
"displayMode" : "BASIC",
"name" : "retryAttempts",
"type" : "NUMBER",
"defaultValue" : -1
}, {
"fieldName" : "ludicrousMode",
"description" : "Ludicrous mode may significantly improve performance, but metrics will be limited",
"max" : 9223372036854775807,
"label" : "Enable Ludicrous Mode",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : {
"executionMode" : [ "STREAMING", "BATCH" ]
},
"dependsOn" : "",
"group" : "",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 40,
"displayMode" : "BASIC",
"name" : "ludicrousMode",
"type" : "BOOLEAN",
"defaultValue" : false
}, {
"fieldName" : "ludicrousModeInputCount",
"description" : "Collects and displays input metrics. Can result in rereading data unless origins are configured to cache data",
"max" : 9223372036854775807,
"label" : "Collect Input Metrics",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : {
"executionMode" : [ "STREAMING", "BATCH" ],
"ludicrousMode" : [ true ]
},
"dependsOn" : "",
"group" : "",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 50,
"displayMode" : "BASIC",
"name" : "ludicrousModeInputCount",
"type" : "BOOLEAN",
"defaultValue" : false
}, {
"fieldName" : "advancedErrorHandling",
"description" : "Reports the record that generates an error, when possible. Supported in single-origin pipelines",
"max" : 9223372036854775807,
"label" : "Advanced Error Handling",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : {
"executionMode" : [ "STREAMING", "BATCH" ]
},
"dependsOn" : "",
"group" : "",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 60,
"displayMode" : "BASIC",
"name" : "advancedErrorHandling",
"type" : "BOOLEAN",
"defaultValue" : false
}, {
"fieldName" : "notifyOnStates",
"description" : "Notifies via email when pipeline gets to the specified states",
"max" : 9223372036854775807,
"label" : "Notify on Pipeline State Changes",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : {
"executionMode" : [ "STANDALONE", "CLUSTER_BATCH", "CLUSTER_YARN_STREAMING", "STREAMING", "CLUSTER_MESOS_STREAMING", "BATCH" ]
},
"dependsOn" : "",
"group" : "NOTIFICATIONS",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "Running", "Start Error", "Run Error", "Running Error", "Stopped", "Finished", "Disconnected", "Connecting", "Stop Error" ],
"values" : [ "RUNNING", "START_ERROR", "RUN_ERROR", "RUNNING_ERROR", "STOPPED", "FINISHED", "DISCONNECTED", "CONNECTING", "STOP_ERROR" ],
"configDefinitions" : null,
"modelType" : "MULTI_VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.datacollector.config.PipelineStateChooserValues",
"filteringConfig" : null
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 75,
"displayMode" : "BASIC",
"name" : "notifyOnStates",
"type" : "MODEL",
"defaultValue" : [ "RUN_ERROR", "STOPPED", "FINISHED" ]
}, {
"fieldName" : "emailIDs",
"description" : "Email Addresses",
"max" : 9223372036854775807,
"label" : "Email IDs",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : {
"executionMode" : [ "STANDALONE", "CLUSTER_BATCH", "CLUSTER_YARN_STREAMING", "STREAMING", "CLUSTER_MESOS_STREAMING", "BATCH" ]
},
"dependsOn" : "",
"group" : "NOTIFICATIONS",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 76,
"displayMode" : "BASIC",
"name" : "emailIDs",
"type" : "LIST",
"defaultValue" : [ ]
}, {
"fieldName" : "constants",
"description" : "",
"max" : 9223372036854775807,
"label" : "Parameters",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : { },
"dependsOn" : "",
"group" : "PARAMETERS",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 80,
"displayMode" : "BASIC",
"name" : "constants",
"type" : "MAP",
"defaultValue" : [ ]
}, {
"fieldName" : "badRecordsHandling",
"description" : "",
"max" : 9223372036854775807,
"label" : "Error Records",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"executionMode" : [ "STANDALONE", "CLUSTER_BATCH", "EMR_BATCH", "CLUSTER_YARN_STREAMING", "EDGE", "CLUSTER_MESOS_STREAMING" ]
},
"dependsOn" : "",
"group" : "BAD_RECORDS",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "Discard (Library: Basic)", "Send Response to Origin (Library: Basic)", "Write to Amazon S3 (Library: Amazon Web Services 1.11.123)", "Write to Another Pipeline (Library: Basic)", "Write to Azure Event Hub (Library: Azure)", "Write to Elasticsearch (Library: Elasticsearch 5.2.0)", "Write to File (Library: Basic)", "Write to Google Cloud Storage (Library: Google Cloud)", "Write to Google Pub Sub (Library: Google Cloud)", "Write to Kafka (Library: Apache Kafka 1.1.0)", "Write to Kafka (Library: CDH 6.3.0)", "Write to Kafka (Library: Apache Kafka 2.0.0)", "Write to Kafka (Library: CDH 6.0.1)", "Write to Kafka (Library: HDP 3.1.0)", "Write to Kafka (Library: CDH Kafka 3.1.0)", "Write to Kafka (Library: Apache Kafka 1.0.0)", "Write to Kafka (Library: CDH 6.1.1)", "Write to Kafka (Library: CDH 6.2.0)", "Write to Kinesis (Library: Amazon Kinesis)", "Write to MQTT (Library: Basic)" ],
"values" : [ "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1", "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_tooriginresponse_ToErrorOriginResponseDTarget::1", "streamsets-datacollector-aws-lib::com_streamsets_pipeline_stage_destination_s3_ToErrorAmazonS3DTarget::11", "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_sdcipc_ToErrorSdcIpcDTarget::2", "streamsets-datacollector-azure-lib::com_streamsets_pipeline_stage_destination_eventhubs_ToErrorEventHubProducerDTarget::1", "streamsets-datacollector-elasticsearch_5-lib::com_streamsets_pipeline_stage_destination_elasticsearch_ToErrorElasticSearchDTarget::10", "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_recordstolocalfilesystem_ToErrorLocalFSDTarget::1", "streamsets-datacollector-google-cloud-lib::com_streamsets_pipeline_stage_cloudstorage_destination_ToErrorGoogleCloudStorageDTarget::1", "streamsets-datacollector-google-cloud-lib::com_streamsets_pipeline_stage_pubsub_destination_ToErrorPubSubDTarget::2", "streamsets-datacollector-apache-kafka_1_1-lib::com_streamsets_pipeline_stage_destination_kafka_ToErrorKafkaDTarget::6", "streamsets-datacollector-cdh_6_3-lib::com_streamsets_pipeline_stage_destination_kafka_ToErrorKafkaDTarget::6", "streamsets-datacollector-apache-kafka_2_0-lib::com_streamsets_pipeline_stage_destination_kafka_ToErrorKafkaDTarget::6", "streamsets-datacollector-cdh_6_0-lib::com_streamsets_pipeline_stage_destination_kafka_ToErrorKafkaDTarget::6", "streamsets-datacollector-hdp_3_1-lib::com_streamsets_pipeline_stage_destination_kafka_ToErrorKafkaDTarget::6", "streamsets-datacollector-cdh_kafka_3_1-lib::com_streamsets_pipeline_stage_destination_kafka_ToErrorKafkaDTarget::6", "streamsets-datacollector-apache-kafka_1_0-lib::com_streamsets_pipeline_stage_destination_kafka_ToErrorKafkaDTarget::6", "streamsets-datacollector-cdh_6_1-lib::com_streamsets_pipeline_stage_destination_kafka_ToErrorKafkaDTarget::6", "streamsets-datacollector-cdh_6_2-lib::com_streamsets_pipeline_stage_destination_kafka_ToErrorKafkaDTarget::6", "streamsets-datacollector-kinesis-lib::com_streamsets_pipeline_stage_destination_kinesis_ToErrorKinesisDTarget::8", "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_mqtt_ToErrorMqttClientDTarget::4" ],
"configDefinitions" : null,
"modelType" : "VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.datacollector.config.ErrorHandlingChooserValues",
"filteringConfig" : ""
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 90,
"displayMode" : "BASIC",
"name" : "badRecordsHandling",
"type" : "MODEL",
"defaultValue" : null
}, {
"fieldName" : "errorRecordPolicy",
"description" : "Determines which variation of the record is sent to error.",
"max" : 9223372036854775807,
"label" : "Error Record Policy",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"executionMode" : [ "STANDALONE", "CLUSTER_BATCH", "EMR_BATCH", "CLUSTER_YARN_STREAMING", "EDGE", "CLUSTER_MESOS_STREAMING" ]
},
"dependsOn" : "",
"group" : "BAD_RECORDS",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "Original record as it was generated by the origin.", "Record as it was seen by the stage that sent it to error stream." ],
"values" : [ "ORIGINAL_RECORD", "STAGE_RECORD" ],
"configDefinitions" : null,
"modelType" : "VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.datacollector.config.ErrorRecordPolicyChooserValues",
"filteringConfig" : ""
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 93,
"displayMode" : "BASIC",
"name" : "errorRecordPolicy",
"type" : "MODEL",
"defaultValue" : "ORIGINAL_RECORD"
}, {
"fieldName" : "statsAggregatorStage",
"description" : "",
"max" : 9223372036854775807,
"label" : "Statistics Aggregator",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : {
"executionMode" : [ "STANDALONE", "CLUSTER_BATCH", "EMR_BATCH", "CLUSTER_YARN_STREAMING", "STREAMING", "EDGE", "CLUSTER_MESOS_STREAMING", "BATCH" ]
},
"dependsOn" : "",
"group" : "STATS",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "Discard (Library: Basic)", "Write Directly to Control Hub - statistics are not aggregated across Data Collectors (Library: Basic)", "Write to Kafka (Library: Apache Kafka 1.1.0)", "Write to Kafka (Library: CDH 6.3.0)", "Write to Kafka (Library: Apache Kafka 2.0.0)", "Write to Kafka (Library: CDH 6.0.1)", "Write to Kafka (Library: HDP 3.1.0)", "Write to Kafka (Library: CDH Kafka 3.1.0)", "Write to Kafka (Library: Apache Kafka 1.0.0)", "Write to Kafka (Library: CDH 6.1.1)", "Write to Kafka (Library: CDH 6.2.0)", "Write to Kinesis (Library: Amazon Kinesis)", "Write to SDC RPC (Library: Basic)" ],
"values" : [ "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_StatsNullDTarget::1", "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_StatsDpmDirectlyDTarget::1", "streamsets-datacollector-apache-kafka_1_1-lib::com_streamsets_pipeline_stage_destination_kafka_StatsKafkaDTarget::4", "streamsets-datacollector-cdh_6_3-lib::com_streamsets_pipeline_stage_destination_kafka_StatsKafkaDTarget::4", "streamsets-datacollector-apache-kafka_2_0-lib::com_streamsets_pipeline_stage_destination_kafka_StatsKafkaDTarget::4", "streamsets-datacollector-cdh_6_0-lib::com_streamsets_pipeline_stage_destination_kafka_StatsKafkaDTarget::4", "streamsets-datacollector-hdp_3_1-lib::com_streamsets_pipeline_stage_destination_kafka_StatsKafkaDTarget::4", "streamsets-datacollector-cdh_kafka_3_1-lib::com_streamsets_pipeline_stage_destination_kafka_StatsKafkaDTarget::4", "streamsets-datacollector-apache-kafka_1_0-lib::com_streamsets_pipeline_stage_destination_kafka_StatsKafkaDTarget::4", "streamsets-datacollector-cdh_6_1-lib::com_streamsets_pipeline_stage_destination_kafka_StatsKafkaDTarget::4", "streamsets-datacollector-cdh_6_2-lib::com_streamsets_pipeline_stage_destination_kafka_StatsKafkaDTarget::4", "streamsets-datacollector-kinesis-lib::com_streamsets_pipeline_stage_destination_kinesis_StatsKinesisDTarget::8", "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_sdcipc_StatsSdcIpcDTarget::2" ],
"configDefinitions" : null,
"modelType" : "VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.datacollector.config.StatsTargetChooserValues",
"filteringConfig" : ""
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 95,
"displayMode" : "BASIC",
"name" : "statsAggregatorStage",
"type" : "MODEL",
"defaultValue" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_StatsDpmDirectlyDTarget::1"
}, {
"fieldName" : "workerCount",
"description" : "Number of workers. 0 to start as many workers as Kafka partitions for topic.",
"max" : 9223372036854775807,
"label" : "Worker Count",
"required" : true,
"min" : 0,
"dependsOnMap" : {
"executionMode" : [ "CLUSTER_YARN_STREAMING" ]
},
"dependsOn" : "",
"group" : "CLUSTER",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 100,
"displayMode" : "BASIC",
"name" : "workerCount",
"type" : "NUMBER",
"defaultValue" : 0
}, {
"fieldName" : "clusterSlaveMemory",
"description" : "",
"max" : 9223372036854775807,
"label" : "Worker Memory (MB)",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"executionMode" : [ "CLUSTER_BATCH", "EMR_BATCH", "CLUSTER_YARN_STREAMING" ]
},
"dependsOn" : "",
"group" : "CLUSTER",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 150,
"displayMode" : "BASIC",
"name" : "clusterSlaveMemory",
"type" : "NUMBER",
"defaultValue" : 2048
}, {
"fieldName" : "clusterSlaveJavaOpts",
"description" : "Add properties as needed. Changes to default settings are not recommended.",
"max" : 9223372036854775807,
"label" : "Worker Java Options",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"executionMode" : [ "CLUSTER_BATCH", "EMR_BATCH", "CLUSTER_YARN_STREAMING" ]
},
"dependsOn" : "",
"group" : "CLUSTER",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 110,
"displayMode" : "BASIC",
"name" : "clusterSlaveJavaOpts",
"type" : "STRING",
"defaultValue" : "-XX:+UseConcMarkSweepGC -XX:+UseParNewGC -Dlog4j.debug"
}, {
"fieldName" : "clusterLauncherEnv",
"description" : "Sets additional environment variables for the cluster launcher",
"max" : 9223372036854775807,
"label" : "Launcher ENV",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : {
"executionMode" : [ "CLUSTER_BATCH", "CLUSTER_YARN_STREAMING" ]
},
"dependsOn" : "",
"group" : "CLUSTER",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 120,
"displayMode" : "BASIC",
"name" : "clusterLauncherEnv",
"type" : "MAP",
"defaultValue" : [ ]
}, {
"fieldName" : "mesosDispatcherURL",
"description" : "URL for service which launches Mesos framework",
"max" : 9223372036854775807,
"label" : "Mesos Dispatcher URL",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"executionMode" : [ "CLUSTER_MESOS_STREAMING" ]
},
"dependsOn" : "",
"group" : "CLUSTER",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 130,
"displayMode" : "BASIC",
"name" : "mesosDispatcherURL",
"type" : "STRING",
"defaultValue" : null
}, {
"fieldName" : "logLevel",
"description" : "Log level to use for the launched application",
"max" : 9223372036854775807,
"label" : "Log Level",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"executionMode" : [ "EMR_BATCH", "STREAMING", "BATCH" ]
},
"dependsOn" : "",
"group" : "CLUSTER",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "OFF", "FATAL", "ERROR", "WARN", "INFO", "DEBUG", "TRACE" ],
"values" : [ "OFF", "FATAL", "ERROR", "WARN", "INFO", "DEBUG", "TRACE" ],
"configDefinitions" : null,
"modelType" : "VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.datacollector.config.LogLevelChooserValues",
"filteringConfig" : ""
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 140,
"displayMode" : "BASIC",
"name" : "logLevel",
"type" : "MODEL",
"defaultValue" : "INFO"
}, {
"fieldName" : "hdfsS3ConfDir",
"description" : "An SDC resource directory or symbolic link with HDFS/S3 configuration files core-site.xml and hdfs-site.xml",
"max" : 9223372036854775807,
"label" : "Checkpoint Configuration Directory",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"executionMode" : [ "CLUSTER_MESOS_STREAMING" ]
},
"dependsOn" : "",
"group" : "CLUSTER",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 150,
"displayMode" : "BASIC",
"name" : "hdfsS3ConfDir",
"type" : "STRING",
"defaultValue" : null
}, {
"fieldName" : "rateLimit",
"description" : "Maximum number of records per second that should be accepted into the pipeline. Rate is not limited if this is not set, or is set to 0",
"max" : 9223372036854775807,
"label" : "Rate Limit (records / sec)",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : {
"executionMode" : [ "STANDALONE", "CLUSTER_BATCH", "CLUSTER_YARN_STREAMING", "CLUSTER_MESOS_STREAMING" ]
},
"dependsOn" : "",
"group" : "",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 180,
"displayMode" : "BASIC",
"name" : "rateLimit",
"type" : "NUMBER",
"defaultValue" : 0
}, {
"fieldName" : "maxRunners",
"description" : "Maximum number of runners that should be created for this pipeline. Use 0 to not impose limit.",
"max" : 9223372036854775807,
"label" : "Max runners",
"required" : false,
"min" : 0,
"dependsOnMap" : {
"executionMode" : [ "STANDALONE", "CLUSTER_BATCH", "CLUSTER_YARN_STREAMING", "CLUSTER_MESOS_STREAMING" ]
},
"dependsOn" : "",
"group" : "",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 190,
"displayMode" : "BASIC",
"name" : "maxRunners",
"type" : "NUMBER",
"defaultValue" : 0
}, {
"fieldName" : "shouldCreateFailureSnapshot",
"description" : "When selected and the pipeline execution fails with unrecoverable exception, SDC will attempt to createpartial snapshot with records that have not been processed yet.",
"max" : 9223372036854775807,
"label" : "Create Failure Snapshot",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"executionMode" : [ "STANDALONE" ]
},
"dependsOn" : "",
"group" : "",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 200,
"displayMode" : "BASIC",
"name" : "shouldCreateFailureSnapshot",
"type" : "BOOLEAN",
"defaultValue" : true
}, {
"fieldName" : "runnerIdleTIme",
"description" : "When pipeline runners are idle for at least this time, run an empty batch through the runner to process any events or other time-driven functionality. Value -1 will disable this functionality completely.",
"max" : 9223372036854775807,
"label" : "Runner Idle Time (sec)",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"executionMode" : [ "STANDALONE" ]
},
"dependsOn" : "",
"group" : "",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 210,
"displayMode" : "BASIC",
"name" : "runnerIdleTIme",
"type" : "NUMBER",
"defaultValue" : 60
}, {
"fieldName" : "webhookConfigs",
"description" : "Webhooks",
"max" : 9223372036854775807,
"label" : "Webhooks",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"executionMode" : [ "STANDALONE", "CLUSTER_BATCH", "CLUSTER_YARN_STREAMING", "STREAMING", "CLUSTER_MESOS_STREAMING", "BATCH" ]
},
"dependsOn" : "",
"group" : "NOTIFICATIONS",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : null,
"values" : null,
"configDefinitions" : [ {
"fieldName" : "payload",
"description" : "Data that should be included as a part of the Webhook request",
"max" : 9223372036854775807,
"label" : "Payload",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : {
"httpMethod" : [ "POST", "DELETE", "PUT" ]
},
"dependsOn" : "",
"group" : "NOTIFICATIONS",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "application/json",
"lines" : 2,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 240,
"displayMode" : "BASIC",
"name" : "payload",
"type" : "TEXT",
"defaultValue" : "{\n \"text\" : \"Pipeline '{{PIPELINE_TITLE}}' state changed to {{PIPELINE_STATE}} at {{TIME}}. \\n <{{PIPELINE_URL}}|Click here> for details!\",\n \"fields\": [\n \t{\n\t\t\t\"title\": \"Pipeline Status Message\", \n\t\t\t\"value\": \"{{PIPELINE_STATE_MESSAGE}}\",\n \t\t\"short\": true\n\t\t},\n\t\t{\n\t\t\t\"title\": \"Pipeline Input Records Count\", \n\t\t\t\"value\": \"{{PIPELINE_INPUT_RECORDS_COUNT}}\",\n \"short\": true\n\t\t},\n\t\t{\n\t\t\t\"title\": \"Pipeline Output Records Count\", \n\t\t\t\"value\": \"{{PIPELINE_OUTPUT_RECORDS_COUNT}}\",\n \"short\": true\n\t\t},\n\t\t{\n\t\t\t\"title\": \"Pipeline Error Records Count\", \n\t\t\t\"value\": \"{{PIPELINE_ERROR_RECORDS_COUNT}}\",\n \"short\": true\n\t\t},\n\t\t{\n\t\t\t\"title\": \"Pipeline Error Messages Count\", \n\t\t\t\"value\": \"{{PIPELINE_ERROR_MESSAGES_COUNT}}\",\n \"short\": true\n\t\t},\n\t\t{\n\t\t\t\"title\": \"Pipeline Runtime Parameters\", \n\t\t\t\"value\": \"{{PIPELINE_RUNTIME_PARAMETERS}}\",\n \t\"short\": true\n\t\t}\n\t]\n}"
}, {
"fieldName" : "webhookUrl",
"description" : "The Webhook HTTP resource URL",
"max" : 9223372036854775807,
"label" : "Webhook URL",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : { },
"dependsOn" : "",
"group" : "",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 200,
"displayMode" : "BASIC",
"name" : "webhookUrl",
"type" : "STRING",
"defaultValue" : null
}, {
"fieldName" : "headers",
"description" : "Headers to include in the request",
"max" : 9223372036854775807,
"label" : "Headers",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : { },
"dependsOn" : "",
"group" : "NOTIFICATIONS",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 210,
"displayMode" : "BASIC",
"name" : "headers",
"type" : "MAP",
"defaultValue" : [ ]
}, {
"fieldName" : "httpMethod",
"description" : "HTTP method to send",
"max" : 9223372036854775807,
"label" : "HTTP Method",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : { },
"dependsOn" : "",
"group" : "NOTIFICATIONS",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "GET", "PUT", "POST", "DELETE", "HEAD" ],
"values" : [ "GET", "PUT", "POST", "DELETE", "HEAD" ],
"configDefinitions" : null,
"modelType" : "VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.datacollector.config.HttpMethodChooserValues",
"filteringConfig" : ""
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 220,
"displayMode" : "BASIC",
"name" : "httpMethod",
"type" : "MODEL",
"defaultValue" : "POST"
}, {
"fieldName" : "contentType",
"description" : "Content-Type header to be sent with the request; used if that header is not already present",
"max" : 9223372036854775807,
"label" : "Content Type",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : {
"httpMethod" : [ "POST", "DELETE", "PUT" ]
},
"dependsOn" : "",
"group" : "NOTIFICATIONS",
"evaluation" : "EXPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 250,
"displayMode" : "BASIC",
"name" : "contentType",
"type" : "STRING",
"defaultValue" : "application/json"
}, {
"fieldName" : "authType",
"description" : "",
"max" : 9223372036854775807,
"label" : "Authentication Type",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : { },
"dependsOn" : "",
"group" : "NOTIFICATIONS",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "None", "Basic", "Digest", "Universal" ],
"values" : [ "NONE", "BASIC", "DIGEST", "UNIVERSAL" ],
"configDefinitions" : null,
"modelType" : "VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.datacollector.config.AuthenticationTypeChooserValues",
"filteringConfig" : ""
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 260,
"displayMode" : "BASIC",
"name" : "authType",
"type" : "MODEL",
"defaultValue" : "NONE"
}, {
"fieldName" : "username",
"description" : "",
"max" : 9223372036854775807,
"label" : "Username",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"authType" : [ "DIGEST", "BASIC", "UNIVERSAL" ]
},
"dependsOn" : "",
"group" : "NOTIFICATIONS",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61", "63", "64", "65", "66" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 270,
"displayMode" : "BASIC",
"name" : "username",
"type" : "CREDENTIAL",
"defaultValue" : null
}, {
"fieldName" : "password",
"description" : "",
"max" : 9223372036854775807,
"label" : "Password",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"authType" : [ "DIGEST", "BASIC", "UNIVERSAL" ]
},
"dependsOn" : "",
"group" : "NOTIFICATIONS",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61", "63", "64", "65", "66" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 280,
"displayMode" : "BASIC",
"name" : "password",
"type" : "CREDENTIAL",
"defaultValue" : null
} ],
"modelType" : "LIST_BEAN",
"valuesProviderClass" : null,
"filteringConfig" : null
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 210,
"displayMode" : "BASIC",
"name" : "webhookConfigs",
"type" : "MODEL",
"defaultValue" : [ ]
}, {
"fieldName" : "sparkConfigs",
"description" : "Additional Spark Configuration to pass to the spark-submit script, the parameters will be passed as --conf <key>=<value>",
"max" : 9223372036854775807,
"label" : "Extra Spark Configuration",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : {
"executionMode" : [ "CLUSTER_BATCH", "CLUSTER_YARN_STREAMING", "STREAMING", "BATCH" ]
},
"dependsOn" : "",
"group" : "CLUSTER",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 220,
"displayMode" : "BASIC",
"name" : "sparkConfigs",
"type" : "MAP",
"defaultValue" : [ ]
}, {
"fieldName" : "preprocessScript",
"description" : "Scala script to run on the driver before starting the pipeline. Can be used to register user defined functions, etc. Use the 'spark' variable to access the Spark session",
"max" : 9223372036854775807,
"label" : "Preprocessing Script",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : {
"executionMode" : [ "STREAMING", "BATCH" ]
},
"dependsOn" : "",
"group" : "ADVANCED",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/x-scala",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 10,
"displayMode" : "BASIC",
"name" : "preprocessScript",
"type" : "TEXT",
"defaultValue" : "/*\nThe following script define a method\nthat increments an integer by 1 \nand registers it as a UDF with \nthe SparkSession, which can be accessed\nusing the variable named \"spark\":\ndef inc(i: Integer): Integer = {\n i + 1\n}\nspark.udf.register (\"inc\", inc _)\n\n*/"
}, {
"fieldName" : "clusterType",
"description" : "Type of cluster manager used by the Spark cluster",
"max" : 9223372036854775807,
"label" : "Cluster Manager Type",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"executionMode" : [ "STREAMING", "BATCH" ]
},
"dependsOn" : "",
"group" : "CLUSTER",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "None (local)", "Hadoop YARN", "Databricks", "Standalone Spark Cluster", "SQL Server Big Data Cluster", "Apache Spark for HDInsight" ],
"values" : [ "LOCAL", "YARN", "DATABRICKS", "STANDALONE_SPARK_CLUSTER", "SQL_SERVER_BIG_DATA_CLUSTER", "AZURE_HD_INSIGHT" ],
"configDefinitions" : null,
"modelType" : "VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.datacollector.config.SparkClusterTypeChooserValues",
"filteringConfig" : ""
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 101,
"displayMode" : "BASIC",
"name" : "clusterConfig.clusterType",
"type" : "MODEL",
"defaultValue" : "LOCAL"
}, {
"fieldName" : "sparkMasterUrl",
"description" : "Master URL to connect to Spark",
"max" : 9223372036854775807,
"label" : "Master URL",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"clusterConfig.clusterType" : [ "STANDALONE_SPARK_CLUSTER", "LOCAL" ],
"executionMode" : [ "STREAMING", "BATCH" ]
},
"dependsOn" : "",
"group" : "CLUSTER",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 102,
"displayMode" : "BASIC",
"name" : "clusterConfig.sparkMasterUrl",
"type" : "STRING",
"defaultValue" : "local[*]"
}, {
"fieldName" : "deployMode",
"description" : "Mode used to launch the Spark driver process",
"max" : 9223372036854775807,
"label" : "Deployment Mode",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"clusterConfig.clusterType" : [ "STANDALONE_SPARK_CLUSTER", "YARN" ],
"executionMode" : [ "STREAMING", "BATCH" ]
},
"dependsOn" : "",
"group" : "CLUSTER",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "Client", "Cluster" ],
"values" : [ "CLIENT", "CLUSTER" ],
"configDefinitions" : null,
"modelType" : "VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.datacollector.config.SparkDeployModeChooserValues",
"filteringConfig" : ""
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 103,
"displayMode" : "BASIC",
"name" : "clusterConfig.deployMode",
"type" : "MODEL",
"defaultValue" : "CLIENT"
}, {
"fieldName" : "hadoopUserName",
"description" : "Name of the Hadoop user that StreamSets impersonates",
"max" : 9223372036854775807,
"label" : "Hadoop User Name",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : {
"clusterConfig.clusterType" : [ "YARN" ],
"executionMode" : [ "STREAMING", "BATCH" ]
},
"dependsOn" : "",
"group" : "CLUSTER",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 104,
"displayMode" : "BASIC",
"name" : "clusterConfig.hadoopUserName",
"type" : "STRING",
"defaultValue" : null
}, {
"fieldName" : "sparkAppName",
"description" : "Name of the launched Spark application",
"max" : 9223372036854775807,
"label" : "Application Name",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"executionMode" : [ "STREAMING", "BATCH" ]
},
"dependsOn" : "",
"group" : "CLUSTER",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 105,
"displayMode" : "BASIC",
"name" : "clusterConfig.sparkAppName",
"type" : "STRING",
"defaultValue" : "${pipeline:title()}"
}, {
"fieldName" : "stagingDir",
"description" : "Staging directory on the remote system for copying StreamSets resources",
"max" : 9223372036854775807,
"label" : "Staging Directory",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"clusterConfig.clusterType" : [ "SQL_SERVER_BIG_DATA_CLUSTER", "AZURE_HD_INSIGHT", "DATABRICKS" ],
"executionMode" : [ "STREAMING", "BATCH" ]
},
"dependsOn" : "",
"group" : "CLUSTER",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 106,
"displayMode" : "BASIC",
"name" : "clusterConfig.stagingDir",
"type" : "STRING",
"defaultValue" : "/streamsets"
}, {
"fieldName" : "useYarnKerberosKeytab",
"description" : "Indicates that the Spark application should use a principal and keytab for Kerberos authentication",
"max" : 9223372036854775807,
"label" : "Use YARN Kerberos Keytab",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"clusterConfig.clusterType" : [ "YARN" ],
"executionMode" : [ "STREAMING", "BATCH" ]
},
"dependsOn" : "",
"group" : "CLUSTER",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 1000,
"displayMode" : "BASIC",
"name" : "clusterConfig.useYarnKerberosKeytab",
"type" : "BOOLEAN",
"defaultValue" : false
}, {
"fieldName" : "yarnKerberosKeytabSource",
"description" : "Source for the Kerberos keytab used to launch the Spark application",
"max" : 9223372036854775807,
"label" : "Keytab Source",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"clusterConfig.clusterType" : [ "YARN" ],
"executionMode" : [ "STREAMING", "BATCH" ],
"clusterConfig.useYarnKerberosKeytab" : [ true ]
},
"dependsOn" : "",
"group" : "CLUSTER",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "Properties file", "Pipeline configuration" ],
"values" : [ "PROPERTIES_FILE", "PIPELINE" ],
"configDefinitions" : null,
"modelType" : "VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.datacollector.config.KeytabSourceChooserValues",
"filteringConfig" : ""
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 1050,
"displayMode" : "BASIC",
"name" : "clusterConfig.yarnKerberosKeytabSource",
"type" : "MODEL",
"defaultValue" : "PROPERTIES_FILE"
}, {
"fieldName" : "yarnKerberosKeytab",
"description" : "Absolute path to the Kerberos keytab used to launch the Spark application for the pipeline",
"max" : 9223372036854775807,
"label" : "YARN Kerberos Keytab",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"clusterConfig.yarnKerberosKeytabSource" : [ "PIPELINE" ],
"clusterConfig.clusterType" : [ "YARN" ],
"executionMode" : [ "STREAMING", "BATCH" ],
"clusterConfig.useYarnKerberosKeytab" : [ true ]
},
"dependsOn" : "",
"group" : "CLUSTER",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 1100,
"displayMode" : "BASIC",
"name" : "clusterConfig.yarnKerberosKeytab",
"type" : "STRING",
"defaultValue" : null
}, {
"fieldName" : "yarnKerberosPrincipal",
"description" : "Name of the Kerberos principal used to launch the Spark application for the pipeline. Must be present in the keytab specified above.",
"max" : 9223372036854775807,
"label" : "YARN Kerberos Principal",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"clusterConfig.yarnKerberosKeytabSource" : [ "PIPELINE" ],
"clusterConfig.clusterType" : [ "YARN" ],
"executionMode" : [ "STREAMING", "BATCH" ],
"clusterConfig.useYarnKerberosKeytab" : [ true ]
},
"dependsOn" : "",
"group" : "CLUSTER",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 1200,
"displayMode" : "BASIC",
"name" : "clusterConfig.yarnKerberosPrincipal",
"type" : "STRING",
"defaultValue" : "name@DOMAIN"
}, {
"fieldName" : "baseUrl",
"description" : "",
"max" : 9223372036854775807,
"label" : "URL to connect to Databricks",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"clusterConfig.clusterType" : [ "DATABRICKS" ],
"executionMode" : [ "STREAMING", "BATCH" ]
},
"dependsOn" : "",
"group" : "CLUSTER",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 105,
"displayMode" : "BASIC",
"name" : "databricksConfig.baseUrl",
"type" : "STRING",
"defaultValue" : null
}, {
"fieldName" : "credentialType",
"description" : "Type of credential to use to connect to Databricks",
"max" : 9223372036854775807,
"label" : "Credential Type",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"clusterConfig.clusterType" : [ "DATABRICKS" ],
"executionMode" : [ "STREAMING", "BATCH" ]
},
"dependsOn" : "",
"group" : "CLUSTER",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "Username/Password", "Token" ],
"values" : [ "PASSWORD", "TOKEN" ],
"configDefinitions" : null,
"modelType" : "VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.datacollector.config.CredentialTypeChooserValues",
"filteringConfig" : ""
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 106,
"displayMode" : "BASIC",
"name" : "databricksConfig.credentialType",
"type" : "MODEL",
"defaultValue" : null
}, {
"fieldName" : "username",
"description" : "Databricks user name",
"max" : 9223372036854775807,
"label" : "Username",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"clusterConfig.clusterType" : [ "DATABRICKS" ],
"executionMode" : [ "STREAMING", "BATCH" ],
"databricksConfig.credentialType" : [ "PASSWORD" ]
},
"dependsOn" : "",
"group" : "CLUSTER",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61", "63", "64", "65", "66" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 107,
"displayMode" : "BASIC",
"name" : "databricksConfig.username",
"type" : "CREDENTIAL",
"defaultValue" : null
}, {
"fieldName" : "password",
"description" : "Password for the account",
"max" : 9223372036854775807,
"label" : "Password",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"clusterConfig.clusterType" : [ "DATABRICKS" ],
"executionMode" : [ "STREAMING", "BATCH" ],
"databricksConfig.credentialType" : [ "PASSWORD" ]
},
"dependsOn" : "",
"group" : "CLUSTER",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61", "63", "64", "65", "66" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 108,
"displayMode" : "BASIC",
"name" : "databricksConfig.password",
"type" : "CREDENTIAL",
"defaultValue" : null
}, {
"fieldName" : "token",
"description" : "Personal access token for the account",
"max" : 9223372036854775807,
"label" : "Token",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"clusterConfig.clusterType" : [ "DATABRICKS" ],
"executionMode" : [ "STREAMING", "BATCH" ],
"databricksConfig.credentialType" : [ "TOKEN" ]
},
"dependsOn" : "",
"group" : "CLUSTER",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61", "63", "64", "65", "66" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 109,
"displayMode" : "BASIC",
"name" : "databricksConfig.token",
"type" : "CREDENTIAL",
"defaultValue" : null
}, {
"fieldName" : "provisionNewCluster",
"description" : "Provisions a new cluster when the pipeline starts",
"max" : 9223372036854775807,
"label" : "Provision a New Cluster",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"clusterConfig.clusterType" : [ "DATABRICKS" ],
"executionMode" : [ "STREAMING", "BATCH" ]
},
"dependsOn" : "",
"group" : "CLUSTER",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 110,
"displayMode" : "BASIC",
"name" : "databricksConfig.provisionNewCluster",
"type" : "BOOLEAN",
"defaultValue" : true
}, {
"fieldName" : "clusterId",
"description" : "",
"max" : 9223372036854775807,
"label" : "Cluster ID",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"databricksConfig.provisionNewCluster" : [ false ],
"clusterConfig.clusterType" : [ "DATABRICKS" ],
"executionMode" : [ "STREAMING", "BATCH" ]
},
"dependsOn" : "",
"group" : "CLUSTER",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 111,
"displayMode" : "BASIC",
"name" : "databricksConfig.clusterId",
"type" : "STRING",
"defaultValue" : null
}, {
"fieldName" : "clusterConfig",
"description" : "Configuration for the Databricks cluster",
"max" : 9223372036854775807,
"label" : "Cluster Configuration",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : {
"databricksConfig.provisionNewCluster" : [ true ],
"clusterConfig.clusterType" : [ "DATABRICKS" ],
"executionMode" : [ "STREAMING", "BATCH" ]
},
"dependsOn" : "",
"group" : "CLUSTER",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 112,
"displayMode" : "BASIC",
"name" : "databricksConfig.clusterConfig",
"type" : "TEXT",
"defaultValue" : "{\n \"num_workers\": 8,\n \"spark_version\": \"5.3.x-scala2.11\",\n \"node_type_id\": \"i3.xlarge\"\n}"
}, {
"fieldName" : "terminateCluster",
"description" : "Terminates the cluster when the pipeline stops",
"max" : 9223372036854775807,
"label" : "Terminate Cluster",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"databricksConfig.provisionNewCluster" : [ true ],
"clusterConfig.clusterType" : [ "DATABRICKS" ],
"executionMode" : [ "STREAMING", "BATCH" ]
},
"dependsOn" : "",
"group" : "CLUSTER",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 113,
"displayMode" : "BASIC",
"name" : "databricksConfig.terminateCluster",
"type" : "BOOLEAN",
"defaultValue" : false
}, {
"fieldName" : "baseUrl",
"description" : "URL to connect to the Livy Server",
"max" : 9223372036854775807,
"label" : "Livy Endpoint",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"clusterConfig.clusterType" : [ "SQL_SERVER_BIG_DATA_CLUSTER", "AZURE_HD_INSIGHT" ],
"executionMode" : [ "STREAMING", "BATCH" ]
},
"dependsOn" : "",
"group" : "CLUSTER",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 105,
"displayMode" : "BASIC",
"name" : "livyConfig.baseUrl",
"type" : "STRING",
"defaultValue" : "https://localhost:30443/gateway/default/livy/v1/"
}, {
"fieldName" : "username",
"description" : "Livy user name",
"max" : 9223372036854775807,
"label" : "Username",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : {
"clusterConfig.clusterType" : [ "SQL_SERVER_BIG_DATA_CLUSTER", "AZURE_HD_INSIGHT" ],
"executionMode" : [ "STREAMING", "BATCH" ]
},
"dependsOn" : "",
"group" : "CLUSTER",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61", "63", "64", "65", "66" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 107,
"displayMode" : "BASIC",
"name" : "livyConfig.username",
"type" : "CREDENTIAL",
"defaultValue" : null
}, {
"fieldName" : "password",
"description" : "Password for the account",
"max" : 9223372036854775807,
"label" : "Password",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : {
"clusterConfig.clusterType" : [ "SQL_SERVER_BIG_DATA_CLUSTER", "AZURE_HD_INSIGHT" ],
"executionMode" : [ "STREAMING", "BATCH" ]
},
"dependsOn" : "",
"group" : "CLUSTER",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61", "63", "64", "65", "66" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 108,
"displayMode" : "BASIC",
"name" : "livyConfig.password",
"type" : "CREDENTIAL",
"defaultValue" : null
}, {
"fieldName" : "userRegion",
"description" : "AWS region",
"max" : 9223372036854775807,
"label" : "Region",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"executionMode" : [ "EMR_BATCH" ]
},
"dependsOn" : "",
"group" : "EMR",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "US East (Ohio) us-east-2", "US East (N. Virginia) us-east-1", "US West (N. California) us-west-1", "US West (Oregon) us-west-2", "AWS GovCloud (US)", "Aisa Pacific (Tokyo) ap-northeast-1", "Asia Pacific (Seoul) ap-northeast-2", "Asia Pacific (Osaka-Local) ap-northeast-3", "Asia Pacific (Mumbai) ap-south-1", "Asia Pacific (Singapore) ap-southeast-1", "Asia Pacific (Sydney) ap-southeast-2", "Canada (Central) ca-central-1", "China (Beijing) cn-north-1", "China (Ningxia) cn-northwest-1", "EU (Frankfurt) eu-central-1", "EU (Ireland) eu-west-1", "EU (London) eu-west-2", "EU (Paris) eu-west-3", "South America (São Paulo) sa-east-1", "Other - specify" ],
"values" : [ "US_EAST_2", "US_EAST_1", "US_WEST_1", "US_WEST_2", "US_GOV_WEST_1", "AP_NORTHEAST_1", "AP_NORTHEAST_2", "AP_NORTHEAST_3", "AP_SOUTH_1", "AP_SOUTHEAST_1", "AP_SOUTHEAST_2", "CA_CENTRAL_1", "CN_NORTH_1", "CN_NORTHWEST_1", "EU_CENTRAL_1", "EU_WEST_1", "EU_WEST_2", "EU_WEST_3", "SA_EAST_1", "OTHER" ],
"configDefinitions" : null,
"modelType" : "VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.pipeline.lib.aws.AwsRegionChooserValues",
"filteringConfig" : ""
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 100,
"displayMode" : "BASIC",
"name" : "amazonEMRConfig.userRegion",
"type" : "MODEL",
"defaultValue" : null
}, {
"fieldName" : "userRegionCustom",
"description" : "",
"max" : 9223372036854775807,
"label" : "AWS user region (Custom)",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"amazonEMRConfig.userRegion" : [ "OTHER" ],
"executionMode" : [ "EMR_BATCH" ]
},
"dependsOn" : "",
"group" : "EMR",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 105,
"displayMode" : "BASIC",
"name" : "amazonEMRConfig.userRegionCustom",
"type" : "STRING",
"defaultValue" : null
}, {
"fieldName" : "accessKey",
"description" : "",
"max" : 9223372036854775807,
"label" : "AWS access key",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"executionMode" : [ "EMR_BATCH" ]
},
"dependsOn" : "",
"group" : "EMR",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61", "63", "64", "65", "66" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 110,
"displayMode" : "BASIC",
"name" : "amazonEMRConfig.accessKey",
"type" : "CREDENTIAL",
"defaultValue" : null
}, {
"fieldName" : "secretKey",
"description" : "",
"max" : 9223372036854775807,
"label" : "AWS secret key",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"executionMode" : [ "EMR_BATCH" ]
},
"dependsOn" : "",
"group" : "EMR",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61", "63", "64", "65", "66" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 120,
"displayMode" : "BASIC",
"name" : "amazonEMRConfig.secretKey",
"type" : "CREDENTIAL",
"defaultValue" : null
}, {
"fieldName" : "s3StagingUri",
"description" : "S3 Location where the SDC configuration and resources will be uploaded for the execution of the pipeline",
"max" : 9223372036854775807,
"label" : "S3 Staging URI",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"executionMode" : [ "EMR_BATCH" ]
},
"dependsOn" : "",
"group" : "EMR",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 140,
"displayMode" : "BASIC",
"name" : "amazonEMRConfig.s3StagingUri",
"type" : "STRING",
"defaultValue" : null
}, {
"fieldName" : "provisionNewCluster",
"description" : "Provisions a new cluster when the pipeline starts",
"max" : 9223372036854775807,
"label" : "Provision a New Cluster",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"executionMode" : [ "EMR_BATCH" ]
},
"dependsOn" : "",
"group" : "EMR",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 150,
"displayMode" : "BASIC",
"name" : "amazonEMRConfig.provisionNewCluster",
"type" : "BOOLEAN",
"defaultValue" : false
}, {
"fieldName" : "clusterId",
"description" : "",
"max" : 9223372036854775807,
"label" : "Cluster ID",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"amazonEMRConfig.provisionNewCluster" : [ false ],
"executionMode" : [ "EMR_BATCH" ]
},
"dependsOn" : "",
"group" : "EMR",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 160,
"displayMode" : "BASIC",
"name" : "amazonEMRConfig.clusterId",
"type" : "STRING",
"defaultValue" : null
}, {
"fieldName" : "clusterPrefix",
"description" : "",
"max" : 9223372036854775807,
"label" : "Cluster Name Prefix",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"amazonEMRConfig.provisionNewCluster" : [ true ],
"executionMode" : [ "EMR_BATCH" ]
},
"dependsOn" : "",
"group" : "EMR",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 200,
"displayMode" : "BASIC",
"name" : "amazonEMRConfig.clusterPrefix",
"type" : "STRING",
"defaultValue" : null
}, {
"fieldName" : "terminateCluster",
"description" : "Terminates the cluster when the pipeline stops",
"max" : 9223372036854775807,
"label" : "Terminate Cluster",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"amazonEMRConfig.provisionNewCluster" : [ true ],
"executionMode" : [ "EMR_BATCH" ]
},
"dependsOn" : "",
"group" : "EMR",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 210,
"displayMode" : "BASIC",
"name" : "amazonEMRConfig.terminateCluster",
"type" : "BOOLEAN",
"defaultValue" : false
}, {
"fieldName" : "loggingEnabled",
"description" : "Copy cluster log files to S3",
"max" : 9223372036854775807,
"label" : "Logging Enabled",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"amazonEMRConfig.provisionNewCluster" : [ true ],
"executionMode" : [ "EMR_BATCH" ]
},
"dependsOn" : "",
"group" : "EMR",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 220,
"displayMode" : "BASIC",
"name" : "amazonEMRConfig.loggingEnabled",
"type" : "BOOLEAN",
"defaultValue" : true
}, {
"fieldName" : "s3LogUri",
"description" : "",
"max" : 9223372036854775807,
"label" : "S3 Log URI",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"amazonEMRConfig.loggingEnabled" : [ true ],
"amazonEMRConfig.provisionNewCluster" : [ true ],
"executionMode" : [ "EMR_BATCH" ]
},
"dependsOn" : "",
"group" : "EMR",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 230,
"displayMode" : "BASIC",
"name" : "amazonEMRConfig.s3LogUri",
"type" : "STRING",
"defaultValue" : null
}, {
"fieldName" : "enableEMRDebugging",
"description" : "Enable console debugging in EMR",
"max" : 9223372036854775807,
"label" : "Enable Debugging",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"amazonEMRConfig.loggingEnabled" : [ true ],
"amazonEMRConfig.provisionNewCluster" : [ true ],
"executionMode" : [ "EMR_BATCH" ]
},
"dependsOn" : "",
"group" : "EMR",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 240,
"displayMode" : "BASIC",
"name" : "amazonEMRConfig.enableEMRDebugging",
"type" : "BOOLEAN",
"defaultValue" : true
}, {
"fieldName" : "serviceRole",
"description" : "EMR role used by the cluster when provisioning resources and performing other service-level tasks",
"max" : 9223372036854775807,
"label" : "Service Role",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"amazonEMRConfig.provisionNewCluster" : [ true ],
"executionMode" : [ "EMR_BATCH" ]
},
"dependsOn" : "",
"group" : "EMR",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 260,
"displayMode" : "BASIC",
"name" : "amazonEMRConfig.serviceRole",
"type" : "STRING",
"defaultValue" : "EMR_DefaultRole"
}, {
"fieldName" : "jobFlowRole",
"description" : "EMR role for EC2 used by EC2 instances within the cluster",
"max" : 9223372036854775807,
"label" : "Job Flow Role",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"amazonEMRConfig.provisionNewCluster" : [ true ],
"executionMode" : [ "EMR_BATCH" ]
},
"dependsOn" : "",
"group" : "EMR",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 270,
"displayMode" : "BASIC",
"name" : "amazonEMRConfig.jobFlowRole",
"type" : "STRING",
"defaultValue" : "EMR_EC2_DefaultRole"
}, {
"fieldName" : "visibleToAllUsers",
"description" : "",
"max" : 9223372036854775807,
"label" : "Visible to All Users",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : {
"amazonEMRConfig.provisionNewCluster" : [ true ],
"executionMode" : [ "EMR_BATCH" ]
},
"dependsOn" : "",
"group" : "EMR",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 280,
"displayMode" : "BASIC",
"name" : "amazonEMRConfig.visibleToAllUsers",
"type" : "BOOLEAN",
"defaultValue" : true
}, {
"fieldName" : "ec2SubnetId",
"description" : "EC2 subnet identifier to launch the cluster in",
"max" : 9223372036854775807,
"label" : "EC2 subnet ID",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"amazonEMRConfig.provisionNewCluster" : [ true ],
"executionMode" : [ "EMR_BATCH" ]
},
"dependsOn" : "",
"group" : "EMR",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 290,
"displayMode" : "BASIC",
"name" : "amazonEMRConfig.ec2SubnetId",
"type" : "STRING",
"defaultValue" : null
}, {
"fieldName" : "masterSecurityGroup",
"description" : "Security group ID for the master node",
"max" : 9223372036854775807,
"label" : "Master Security Group",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"amazonEMRConfig.provisionNewCluster" : [ true ],
"executionMode" : [ "EMR_BATCH" ]
},
"dependsOn" : "",
"group" : "EMR",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 300,
"displayMode" : "BASIC",
"name" : "amazonEMRConfig.masterSecurityGroup",
"type" : "STRING",
"defaultValue" : null
}, {
"fieldName" : "slaveSecurityGroup",
"description" : "Security group ID for the slave nodes",
"max" : 9223372036854775807,
"label" : "Slave Security Group",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"amazonEMRConfig.provisionNewCluster" : [ true ],
"executionMode" : [ "EMR_BATCH" ]
},
"dependsOn" : "",
"group" : "EMR",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 310,
"displayMode" : "BASIC",
"name" : "amazonEMRConfig.slaveSecurityGroup",
"type" : "STRING",
"defaultValue" : null
}, {
"fieldName" : "instanceCount",
"description" : "",
"max" : 9223372036854775807,
"label" : "Instance Count",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"amazonEMRConfig.provisionNewCluster" : [ true ],
"executionMode" : [ "EMR_BATCH" ]
},
"dependsOn" : "",
"group" : "EMR",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 320,
"displayMode" : "BASIC",
"name" : "amazonEMRConfig.instanceCount",
"type" : "NUMBER",
"defaultValue" : 2
}, {
"fieldName" : "masterInstanceType",
"description" : "",
"max" : 9223372036854775807,
"label" : "Master Instance Type",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"amazonEMRConfig.provisionNewCluster" : [ true ],
"executionMode" : [ "EMR_BATCH" ]
},
"dependsOn" : "",
"group" : "EMR",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "m4.large", "m4.xlarge", "m4.2xlarge", "m4.4xlarge", "m4.10xlarge", "m4.16xlarge", "c4.large", "c4.xlarge", "c4.2xlarge", "c4.4xlarge", "c4.8xlarge", "r3.xlarge", "r3.2xlarge", "r3.4xlarge", "r3.8xlarge", "r4.xlarge", "r4.2xlarge", "r4.4xlarge", "r4.8xlarge", "r4.16xlarge", "i3.xlarge", "i3.2xlarge", "i3.4xlarge", "i3.8xlarge", "i3.16xlarge", "d2.xlarge", "d2.2xlarge", "d2.4xlarge", "d2.8xlarge", "p2.xlarge", "p2.8xlarge", "p2.16xlarge", "p3.2xlarge", "p3.8xlarge", "p3.16xlarge", "Custom" ],
"values" : [ "M4_LARGE", "M4_XLARGE", "M4_2XLARGE", "M4_4XLARGE", "M4_10XLARGE", "M4_16XLARGE", "C4_LARGE", "C4_XLARGE", "C4_2XLARGE", "C4_4XLARGE", "C4_8XLARGE", "R3_XLARGE", "R3_2XLARGE", "R3_4XLARGE", "R3_8XLARGE", "R4_XLARGE", "R4_2XLARGE", "R4_4XLARGE", "R4_8XLARGE", "R4_16XLARGE", "I3_XLARGE", "I3_2XLARGE", "I3_4XLARGE", "I3_8XLARGE", "I3_16XLARGE", "D2_XLARGE", "D2_2XLARGE", "D2_4XLARGE", "D2_8XLARGE", "P2_XLARGE", "P2_8XLARGE", "P2_16XLARGE", "P3_2XLARGE", "P3_8XLARGE", "P3_16XLARGE", "OTHER" ],
"configDefinitions" : null,
"modelType" : "VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.pipeline.lib.aws.AwsInstanceTypeChooserValues",
"filteringConfig" : ""
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 330,
"displayMode" : "BASIC",
"name" : "amazonEMRConfig.masterInstanceType",
"type" : "MODEL",
"defaultValue" : null
}, {
"fieldName" : "masterInstanceTypeCustom",
"description" : "",
"max" : 9223372036854775807,
"label" : "Master Instance type (Custom)",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"amazonEMRConfig.provisionNewCluster" : [ true ],
"amazonEMRConfig.masterInstanceType" : [ "OTHER" ],
"executionMode" : [ "EMR_BATCH" ]
},
"dependsOn" : "",
"group" : "EMR",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 340,
"displayMode" : "BASIC",
"name" : "amazonEMRConfig.masterInstanceTypeCustom",
"type" : "STRING",
"defaultValue" : null
}, {
"fieldName" : "slaveInstanceType",
"description" : "",
"max" : 9223372036854775807,
"label" : "Slave Instance Type",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"amazonEMRConfig.provisionNewCluster" : [ true ],
"executionMode" : [ "EMR_BATCH" ]
},
"dependsOn" : "",
"group" : "EMR",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "m4.large", "m4.xlarge", "m4.2xlarge", "m4.4xlarge", "m4.10xlarge", "m4.16xlarge", "c4.large", "c4.xlarge", "c4.2xlarge", "c4.4xlarge", "c4.8xlarge", "r3.xlarge", "r3.2xlarge", "r3.4xlarge", "r3.8xlarge", "r4.xlarge", "r4.2xlarge", "r4.4xlarge", "r4.8xlarge", "r4.16xlarge", "i3.xlarge", "i3.2xlarge", "i3.4xlarge", "i3.8xlarge", "i3.16xlarge", "d2.xlarge", "d2.2xlarge", "d2.4xlarge", "d2.8xlarge", "p2.xlarge", "p2.8xlarge", "p2.16xlarge", "p3.2xlarge", "p3.8xlarge", "p3.16xlarge", "Custom" ],
"values" : [ "M4_LARGE", "M4_XLARGE", "M4_2XLARGE", "M4_4XLARGE", "M4_10XLARGE", "M4_16XLARGE", "C4_LARGE", "C4_XLARGE", "C4_2XLARGE", "C4_4XLARGE", "C4_8XLARGE", "R3_XLARGE", "R3_2XLARGE", "R3_4XLARGE", "R3_8XLARGE", "R4_XLARGE", "R4_2XLARGE", "R4_4XLARGE", "R4_8XLARGE", "R4_16XLARGE", "I3_XLARGE", "I3_2XLARGE", "I3_4XLARGE", "I3_8XLARGE", "I3_16XLARGE", "D2_XLARGE", "D2_2XLARGE", "D2_4XLARGE", "D2_8XLARGE", "P2_XLARGE", "P2_8XLARGE", "P2_16XLARGE", "P3_2XLARGE", "P3_8XLARGE", "P3_16XLARGE", "OTHER" ],
"configDefinitions" : null,
"modelType" : "VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.pipeline.lib.aws.AwsInstanceTypeChooserValues",
"filteringConfig" : ""
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 350,
"displayMode" : "BASIC",
"name" : "amazonEMRConfig.slaveInstanceType",
"type" : "MODEL",
"defaultValue" : null
}, {
"fieldName" : "slaveInstanceTypeCustom",
"description" : "",
"max" : 9223372036854775807,
"label" : "Slave Instance type (Custom)",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"amazonEMRConfig.provisionNewCluster" : [ true ],
"executionMode" : [ "EMR_BATCH" ],
"amazonEMRConfig.slaveInstanceType" : [ "OTHER" ]
},
"dependsOn" : "",
"group" : "EMR",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 360,
"displayMode" : "BASIC",
"name" : "amazonEMRConfig.slaveInstanceTypeCustom",
"type" : "STRING",
"defaultValue" : null
} ],
"configGroupDefinition" : {
"classNameToGroupsMap" : {
"com.streamsets.datacollector.config.PipelineGroups" : [ "CLUSTER", "PARAMETERS", "NOTIFICATIONS", "BAD_RECORDS", "STATS", "EMR", "ADVANCED" ]
},
"groupNameToLabelMapList" : [ {
"name" : "CLUSTER",
"label" : "Cluster"
}, {
"name" : "PARAMETERS",
"label" : "Parameters"
}, {
"name" : "NOTIFICATIONS",
"label" : "Notifications"
}, {
"name" : "BAD_RECORDS",
"label" : "Error Records"
}, {
"name" : "STATS",
"label" : "Statistics"
}, {
"name" : "EMR",
"label" : "EMR"
}, {
"name" : "ADVANCED",
"label" : "Advanced"
} ]
}
} ],
"pipelineFragment" : null,
"pipelineRules" : [ {
"configDefinitions" : [ {
"fieldName" : "emailIDs",
"description" : "Email Addresses",
"max" : 9223372036854775807,
"label" : "Email IDs",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : { },
"dependsOn" : "",
"group" : "NOTIFICATIONS",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 76,
"displayMode" : "BASIC",
"name" : "emailIDs",
"type" : "LIST",
"defaultValue" : [ ]
}, {
"fieldName" : "webhookConfigs",
"description" : "Webhooks",
"max" : 9223372036854775807,
"label" : "Webhooks",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : { },
"dependsOn" : "",
"group" : "NOTIFICATIONS",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : null,
"values" : null,
"configDefinitions" : [ {
"fieldName" : "payload",
"description" : "Data that should be included as a part of the Webhook request",
"max" : 9223372036854775807,
"label" : "Payload",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : {
"httpMethod" : [ "POST", "DELETE", "PUT" ]
},
"dependsOn" : "",
"group" : "WEBHOOK",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "application/json",
"lines" : 2,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 240,
"displayMode" : "BASIC",
"name" : "payload",
"type" : "TEXT",
"defaultValue" : "{\n \"text\" : \"{{ALERT_TEXT}}. The alert, {{ALERT_NAME}}, was triggered for pipeline '{{PIPELINE_TITLE}}'. \\n The threshold of {{ALERT_VALUE}} records, for the following condition was reached at {{TIME}}: {{ALERT_CONDITION}} \\n <{{PIPELINE_URL}}|Click here> for details!\"\n}"
}, {
"fieldName" : "webhookUrl",
"description" : "The Webhook HTTP resource URL",
"max" : 9223372036854775807,
"label" : "Webhook URL",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : { },
"dependsOn" : "",
"group" : "",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 200,
"displayMode" : "BASIC",
"name" : "webhookUrl",
"type" : "STRING",
"defaultValue" : null
}, {
"fieldName" : "headers",
"description" : "Headers to include in the request",
"max" : 9223372036854775807,
"label" : "Headers",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : { },
"dependsOn" : "",
"group" : "NOTIFICATIONS",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 210,
"displayMode" : "BASIC",
"name" : "headers",
"type" : "MAP",
"defaultValue" : [ ]
}, {
"fieldName" : "httpMethod",
"description" : "HTTP method to send",
"max" : 9223372036854775807,
"label" : "HTTP Method",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : { },
"dependsOn" : "",
"group" : "NOTIFICATIONS",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "GET", "PUT", "POST", "DELETE", "HEAD" ],
"values" : [ "GET", "PUT", "POST", "DELETE", "HEAD" ],
"configDefinitions" : null,
"modelType" : "VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.datacollector.config.HttpMethodChooserValues",
"filteringConfig" : ""
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 220,
"displayMode" : "BASIC",
"name" : "httpMethod",
"type" : "MODEL",
"defaultValue" : "POST"
}, {
"fieldName" : "contentType",
"description" : "Content-Type header to be sent with the request; used if that header is not already present",
"max" : 9223372036854775807,
"label" : "Content Type",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : {
"httpMethod" : [ "POST", "DELETE", "PUT" ]
},
"dependsOn" : "",
"group" : "NOTIFICATIONS",
"evaluation" : "EXPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 250,
"displayMode" : "BASIC",
"name" : "contentType",
"type" : "STRING",
"defaultValue" : "application/json"
}, {
"fieldName" : "authType",
"description" : "",
"max" : 9223372036854775807,
"label" : "Authentication Type",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : { },
"dependsOn" : "",
"group" : "NOTIFICATIONS",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "None", "Basic", "Digest", "Universal" ],
"values" : [ "NONE", "BASIC", "DIGEST", "UNIVERSAL" ],
"configDefinitions" : null,
"modelType" : "VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.datacollector.config.AuthenticationTypeChooserValues",
"filteringConfig" : ""
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 260,
"displayMode" : "BASIC",
"name" : "authType",
"type" : "MODEL",
"defaultValue" : "NONE"
}, {
"fieldName" : "username",
"description" : "",
"max" : 9223372036854775807,
"label" : "Username",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"authType" : [ "DIGEST", "BASIC", "UNIVERSAL" ]
},
"dependsOn" : "",
"group" : "NOTIFICATIONS",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61", "63", "64", "65", "66" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 270,
"displayMode" : "BASIC",
"name" : "username",
"type" : "CREDENTIAL",
"defaultValue" : null
}, {
"fieldName" : "password",
"description" : "",
"max" : 9223372036854775807,
"label" : "Password",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"authType" : [ "DIGEST", "BASIC", "UNIVERSAL" ]
},
"dependsOn" : "",
"group" : "NOTIFICATIONS",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61", "63", "64", "65", "66" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 280,
"displayMode" : "BASIC",
"name" : "password",
"type" : "CREDENTIAL",
"defaultValue" : null
} ],
"modelType" : "LIST_BEAN",
"valuesProviderClass" : null,
"filteringConfig" : null
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 200,
"displayMode" : "BASIC",
"name" : "webhookConfigs",
"type" : "MODEL",
"defaultValue" : [ ]
} ],
"configGroupDefinition" : {
"classNameToGroupsMap" : {
"com.streamsets.datacollector.config.PipelineRulesGroups" : [ "NOTIFICATIONS" ]
},
"groupNameToLabelMapList" : [ {
"name" : "NOTIFICATIONS",
"label" : "Notifications"
} ]
}
} ],
"stages" : [ {
"icon" : "dev.png",
"version" : "5",
"privateClassLoader" : false,
"description" : "Generates records with the specified field names based on the selected data type. For development only.",
"services" : [ ],
"eventDefs" : [ ],
"library" : "streamsets-datacollector-dev-lib",
"label" : "Dev Data Generator",
"configDefinitions" : [ {
"fieldName" : "dataGenConfigs",
"description" : "Fields to generate of the indicated type",
"max" : 9223372036854775807,
"label" : "Fields to Generate",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : { },
"dependsOn" : "",
"group" : "DATA_GENERATOR",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : null,
"values" : null,
"configDefinitions" : [ {
"fieldName" : "field",
"description" : "",
"max" : 9223372036854775807,
"label" : "Field Name",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : { },
"dependsOn" : "",
"group" : "",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 0,
"displayMode" : "BASIC",
"name" : "field",
"type" : "STRING",
"defaultValue" : null
}, {
"fieldName" : "type",
"description" : "",
"max" : 9223372036854775807,
"label" : "Field Type",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : { },
"dependsOn" : "",
"group" : "",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "STRING", "INTEGER", "LONG", "FLOAT", "DOUBLE", "DATE", "DATETIME", "ZONED_DATETIME", "TIME", "BOOLEAN", "DECIMAL", "BYTE_ARRAY", "LONG_SEQUENCE", "ADDRESS_FULL_ADDRESS", "ADDRESS_BUILDING_NUMBER", "ADDRESS_STREET_ADDRESS", "ADDRESS_CITY", "ADDRESS_STATE", "ADDRESS_COUNTRY", "ADDRESS_LATITUDE", "ADDRESS_LONGITUDE", "APP_NAME", "APP_AUTHOR", "APP_VERSION", "ARTIST_NAME", "BOOK_TITLE", "BOOK_AUTHOR", "BOOK_GENRE", "BOOK_PUBLISHER", "BUSINESS_CREDIT_CARD_NUMBER", "BUSINESS_CREDIT_CARD_EXPIRY", "BUSINESS_CREDIT_CARD_TYPE", "CAT_NAME", "CAT_BREED", "CAT_REGISTRY", "CODE_ASIN", "CODE_IMEI", "CODE_ISBN10", "CODE_ISBN13", "COLOR", "COMMERCE_DEPARTMENT", "COMMERCE_COLOR", "COMMERCE_MATERIAL", "COMMERCE_PRICE", "COMMERCE_PRODUCT_NAME", "COMMERCE_PROMOTION_CODE", "COMPANY_NAME", "COMPANY_INDUSTRY", "COMPANY_BUZZWORD", "COMPANY_URL", "CRYPTO_MD5", "CRYPTO_SHA1", "CRYPTO_SHA256", "CRYPTO_SHA512", "DEMOGRAPHIC", "EDUCATOR", "EMAIL", "FILE", "FINANCE", "FOOD", "GAMEOFTHRONES", "HACKER", "IDNUMBER", "INTERNET", "LOREM", "MUSIC", "NAME", "PHONENUMBER", "POKEMON", "RACE", "SEX", "SHAKESPEARE", "SLACKEMOJI", "SPACE", "SSN", "STOCK", "SUPERHERO", "TEAM", "UNIVERSITY" ],
"values" : [ "STRING", "INTEGER", "LONG", "FLOAT", "DOUBLE", "DATE", "DATETIME", "ZONED_DATETIME", "TIME", "BOOLEAN", "DECIMAL", "BYTE_ARRAY", "LONG_SEQUENCE", "ADDRESS_FULL_ADDRESS", "ADDRESS_BUILDING_NUMBER", "ADDRESS_STREET_ADDRESS", "ADDRESS_CITY", "ADDRESS_STATE", "ADDRESS_COUNTRY", "ADDRESS_LATITUDE", "ADDRESS_LONGITUDE", "APP_NAME", "APP_AUTHOR", "APP_VERSION", "ARTIST_NAME", "BOOK_TITLE", "BOOK_AUTHOR", "BOOK_GENRE", "BOOK_PUBLISHER", "BUSINESS_CREDIT_CARD_NUMBER", "BUSINESS_CREDIT_CARD_EXPIRY", "BUSINESS_CREDIT_CARD_TYPE", "CAT_NAME", "CAT_BREED", "CAT_REGISTRY", "CODE_ASIN", "CODE_IMEI", "CODE_ISBN10", "CODE_ISBN13", "COLOR", "COMMERCE_DEPARTMENT", "COMMERCE_COLOR", "COMMERCE_MATERIAL", "COMMERCE_PRICE", "COMMERCE_PRODUCT_NAME", "COMMERCE_PROMOTION_CODE", "COMPANY_NAME", "COMPANY_INDUSTRY", "COMPANY_BUZZWORD", "COMPANY_URL", "CRYPTO_MD5", "CRYPTO_SHA1", "CRYPTO_SHA256", "CRYPTO_SHA512", "DEMOGRAPHIC", "EDUCATOR", "EMAIL", "FILE", "FINANCE", "FOOD", "GAMEOFTHRONES", "HACKER", "IDNUMBER", "INTERNET", "LOREM", "MUSIC", "NAME", "PHONENUMBER", "POKEMON", "RACE", "SEX", "SHAKESPEARE", "SLACKEMOJI", "SPACE", "SSN", "STOCK", "SUPERHERO", "TEAM", "UNIVERSITY" ],
"configDefinitions" : null,
"modelType" : "VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.pipeline.stage.devtest.TypeChooserValueProvider",
"filteringConfig" : ""
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 0,
"displayMode" : "BASIC",
"name" : "type",
"type" : "MODEL",
"defaultValue" : "STRING"
}, {
"fieldName" : "precision",
"description" : "Precision of the generated decimal.",
"max" : 9223372036854775807,
"label" : "Precision",
"required" : true,
"min" : 0,
"dependsOnMap" : {
"type" : [ "DECIMAL" ]
},
"dependsOn" : "",
"group" : "",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 0,
"displayMode" : "BASIC",
"name" : "precision",
"type" : "NUMBER",
"defaultValue" : 10
}, {
"fieldName" : "scale",
"description" : "Scale of the generated decimal.",
"max" : 9223372036854775807,
"label" : "scale",
"required" : true,
"min" : 0,
"dependsOnMap" : {
"type" : [ "DECIMAL" ]
},
"dependsOn" : "",
"group" : "",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 0,
"displayMode" : "BASIC",
"name" : "scale",
"type" : "NUMBER",
"defaultValue" : 2
} ],
"modelType" : "LIST_BEAN",
"valuesProviderClass" : null,
"filteringConfig" : null
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 0,
"displayMode" : "BASIC",
"name" : "dataGenConfigs",
"type" : "MODEL",
"defaultValue" : [ ]
}, {
"fieldName" : "rootFieldType",
"description" : "Field Type for root object",
"max" : 9223372036854775807,
"label" : "Root Field Type",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : { },
"dependsOn" : "",
"group" : "DATA_GENERATOR",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "MAP", "LIST_MAP" ],
"values" : [ "MAP", "LIST_MAP" ],
"configDefinitions" : null,
"modelType" : "VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.pipeline.stage.devtest.RootTypeChooserValueProvider",
"filteringConfig" : ""
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 0,
"displayMode" : "BASIC",
"name" : "rootFieldType",
"type" : "MODEL",
"defaultValue" : "MAP"
}, {
"fieldName" : "headerAttributes",
"description" : "Attributes to be put in the generated record header",
"max" : 9223372036854775807,
"label" : "Header Attributes",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : { },
"dependsOn" : "",
"group" : "DATA_GENERATOR",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 0,
"displayMode" : "BASIC",
"name" : "headerAttributes",
"type" : "MAP",
"defaultValue" : null
}, {
"fieldName" : "delay",
"description" : "Milliseconds to wait before sending the next batch",
"max" : 2147483647,
"label" : "Delay Between Batches",
"required" : true,
"min" : 0,
"dependsOnMap" : { },
"dependsOn" : "",
"group" : "DATA_GENERATOR",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 0,
"displayMode" : "BASIC",
"name" : "delay",
"type" : "NUMBER",
"defaultValue" : 1000
}, {
"fieldName" : "batchSize",
"description" : "Number of records that will be generated for single batch.",
"max" : 2147483647,
"label" : "Batch size",
"required" : true,
"min" : 1,
"dependsOnMap" : { },
"dependsOn" : "",
"group" : "DATA_GENERATOR",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 0,
"displayMode" : "BASIC",
"name" : "batchSize",
"type" : "NUMBER",
"defaultValue" : 1000
}, {
"fieldName" : "numThreads",
"description" : "Number of concurrent threads that will be generating data in parallel.",
"max" : 2147483647,
"label" : "Number of Threads",
"required" : true,
"min" : 1,
"dependsOnMap" : { },
"dependsOn" : "",
"group" : "DATA_GENERATOR",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 0,
"displayMode" : "BASIC",
"name" : "numThreads",
"type" : "NUMBER",
"defaultValue" : 1
}, {
"fieldName" : "eventName",
"description" : "Name of event that should be used when generating events.",
"max" : 9223372036854775807,
"label" : "Event name",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : { },
"dependsOn" : "",
"group" : "DATA_GENERATOR",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 0,
"displayMode" : "BASIC",
"name" : "eventName",
"type" : "STRING",
"defaultValue" : "generated-event"
}, {
"fieldName" : "stageOnRecordError",
"description" : "Action to take with records sent to error",
"max" : 9223372036854775807,
"label" : "On Record Error",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : { },
"dependsOn" : "",
"group" : "",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "Discard", "Send to Error", "Stop Pipeline" ],
"values" : [ "DISCARD", "TO_ERROR", "STOP_PIPELINE" ],
"configDefinitions" : null,
"modelType" : "VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.datacollector.config.OnRecordErrorChooserValues",
"filteringConfig" : ""
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 30,
"displayMode" : "BASIC",
"name" : "stageOnRecordError",
"type" : "MODEL",
"defaultValue" : "TO_ERROR"
} ],
"configGroupDefinition" : {
"classNameToGroupsMap" : {
"com.streamsets.pipeline.stage.devtest.RandomDataGeneratorGroups" : [ "DATA_GENERATOR" ]
},
"groupNameToLabelMapList" : [ {
"name" : "DATA_GENERATOR",
"label" : "Data Generator"
} ]
},
"libraryLabel" : "Dev (for development only)",
"pipelineLifecycleStage" : false,
"statsAggregatorStage" : false,
"errorStage" : false,
"rawSourceDefinition" : null,
"offsetCommitTrigger" : false,
"variableOutputStreams" : false,
"outputStreams" : 1,
"outputStreamLabelProviderClass" : "com.streamsets.pipeline.api.StageDef$DefaultOutputStreams",
"outputStreamLabels" : null,
"executionModes" : [ "STANDALONE", "EDGE" ],
"resetOffset" : false,
"onlineHelpRefUrl" : "index.html#datacollector/UserGuide/Pipeline_Design/DevStages.html",
"hideStage" : [ ],
"outputStreamsDrivenByConfig" : "",
"sendsResponse" : false,
"beta" : false,
"inputStreams" : -1,
"inputStreamLabelProviderClass" : "com.streamsets.pipeline.api.StageDef$DefaultInputStreams",
"inputStreamLabels" : null,
"bisectable" : false,
"yamlUpgrader" : "upgrader/RandomDataGeneratorSource.yaml",
"name" : "com_streamsets_pipeline_stage_devtest_RandomDataGeneratorSource",
"type" : "SOURCE",
"className" : "com.streamsets.pipeline.stage.devtest.RandomDataGeneratorSource",
"preconditions" : false,
"onRecordError" : true,
"producingEvents" : true
}, {
"icon" : "converter.png",
"version" : "2",
"privateClassLoader" : false,
"description" : "Converts the data type of a field(s)",
"services" : [ ],
"eventDefs" : [ ],
"library" : "streamsets-datacollector-basic-lib",
"label" : "Field Type Converter",
"configDefinitions" : [ {
"fieldName" : "convertBy",
"description" : "Select type of conversion that will be performed.",
"max" : 9223372036854775807,
"label" : "Conversion Method",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : { },
"dependsOn" : "",
"group" : "TYPE_CONVERSION",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "By Field Name", "By Data Type" ],
"values" : [ "BY_FIELD", "BY_TYPE" ],
"configDefinitions" : null,
"modelType" : "VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.pipeline.stage.processor.fieldtypeconverter.ConvertByChooserValues",
"filteringConfig" : ""
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 5,
"displayMode" : "BASIC",
"name" : "convertBy",
"type" : "MODEL",
"defaultValue" : "BY_FIELD"
}, {
"fieldName" : "fieldTypeConverterConfigs",
"description" : "Configures field by names that should be converted",
"max" : 9223372036854775807,
"label" : "",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : {
"convertBy" : [ "BY_FIELD" ]
},
"dependsOn" : "",
"group" : "TYPE_CONVERSION",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : null,
"values" : null,
"configDefinitions" : [ {
"fieldName" : "fields",
"description" : "You can convert multiple fields to the same type",
"max" : 9223372036854775807,
"label" : "Fields to Convert",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : { },
"dependsOn" : "",
"group" : "TYPE_CONVERSION",
"evaluation" : "EXPLICIT",
"elDefs" : null,
"model" : {
"labels" : null,
"values" : null,
"configDefinitions" : null,
"modelType" : "FIELD_SELECTOR_MULTI_VALUE",
"valuesProviderClass" : null,
"filteringConfig" : null
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 10,
"displayMode" : "BASIC",
"name" : "fields",
"type" : "MODEL",
"defaultValue" : [ ]
}, {
"fieldName" : "targetType",
"description" : "Select a compatible data type",
"max" : 9223372036854775807,
"label" : "Convert to Type",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : { },
"dependsOn" : "",
"group" : "",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "BOOLEAN", "CHAR", "BYTE", "SHORT", "INTEGER", "LONG", "FLOAT", "DOUBLE", "DECIMAL", "DATE", "TIME", "DATETIME", "ZONED_DATETIME", "STRING", "BYTE_ARRAY" ],
"values" : [ "BOOLEAN", "CHAR", "BYTE", "SHORT", "INTEGER", "LONG", "FLOAT", "DOUBLE", "DECIMAL", "DATE", "TIME", "DATETIME", "ZONED_DATETIME", "STRING", "BYTE_ARRAY" ],
"configDefinitions" : null,
"modelType" : "VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.pipeline.config.PrimitiveFieldTypeChooserValues",
"filteringConfig" : ""
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 20,
"displayMode" : "BASIC",
"name" : "targetType",
"type" : "MODEL",
"defaultValue" : "INTEGER"
}, {
"fieldName" : "treatInputFieldAsDate",
"description" : "Select to convert input Long to DateTime before converting to a String",
"max" : 9223372036854775807,
"label" : "Treat Input Field as Date",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"targetType" : [ "STRING" ]
},
"dependsOn" : "",
"group" : "TYPE_CONVERSION",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 20,
"displayMode" : "BASIC",
"name" : "treatInputFieldAsDate",
"type" : "BOOLEAN",
"defaultValue" : false
}, {
"fieldName" : "dataLocale",
"description" : "Affects the interpretation of locale sensitive data, such as using the comma as a decimal separator",
"max" : 9223372036854775807,
"label" : "Data Locale",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"targetType" : [ "ZONED_DATETIME", "INTEGER", "LONG", "DOUBLE", "BYTE", "FLOAT", "SHORT", "DECIMAL" ]
},
"dependsOn" : "",
"group" : "TYPE_CONVERSION",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "Albanian", "Albanian (Albania)", "Arabic", "Arabic (Algeria)", "Arabic (Bahrain)", "Arabic (Egypt)", "Arabic (Iraq)", "Arabic (Jordan)", "Arabic (Kuwait)", "Arabic (Lebanon)", "Arabic (Libya)", "Arabic (Morocco)", "Arabic (Oman)", "Arabic (Qatar)", "Arabic (Saudi Arabia)", "Arabic (Sudan)", "Arabic (Syria)", "Arabic (Tunisia)", "Arabic (United Arab Emirates)", "Arabic (Yemen)", "Belarusian", "Belarusian (Belarus)", "Bulgarian", "Bulgarian (Bulgaria)", "Catalan", "Catalan (Spain)", "Chinese", "Chinese (China)", "Chinese (Hong Kong)", "Chinese (Singapore)", "Chinese (Taiwan)", "Croatian", "Croatian (Croatia)", "Czech", "Czech (Czech Republic)", "Danish", "Danish (Denmark)", "Dutch", "Dutch (Belgium)", "Dutch (Netherlands)", "English", "English (Australia)", "English (Canada)", "English (India)", "English (Ireland)", "English (Malta)", "English (New Zealand)", "English (Philippines)", "English (Singapore)", "English (South Africa)", "English (United Kingdom)", "English (United States)", "Estonian", "Estonian (Estonia)", "Finnish", "Finnish (Finland)", "French", "French (Belgium)", "French (Canada)", "French (France)", "French (Luxembourg)", "French (Switzerland)", "German", "German (Austria)", "German (Germany)", "German (Greece)", "German (Luxembourg)", "German (Switzerland)", "Greek", "Greek (Cyprus)", "Greek (Greece)", "Hebrew", "Hebrew (Israel)", "Hindi", "Hindi (India)", "Hungarian", "Hungarian (Hungary)", "Icelandic", "Icelandic (Iceland)", "Indonesian", "Indonesian (Indonesia)", "Irish", "Irish (Ireland)", "Italian", "Italian (Italy)", "Italian (Switzerland)", "Japanese", "Japanese (Japan)", "Japanese (Japan,JP)", "Korean", "Korean (South Korea)", "Latvian", "Latvian (Latvia)", "Lithuanian", "Lithuanian (Lithuania)", "Macedonian", "Macedonian (Macedonia)", "Malay", "Malay (Malaysia)", "Maltese", "Maltese (Malta)", "Norwegian", "Norwegian (Norway)", "Norwegian (Norway,Nynorsk)", "Polish", "Polish (Poland)", "Portuguese", "Portuguese (Brazil)", "Portuguese (Portugal)", "Romanian", "Romanian (Romania)", "Russian", "Russian (Russia)", "Serbian (Latin)", "Serbian (Latin,Bosnia and Herzegovina)", "Serbian (Montenegro)", "Serbian (Serbia and Montenegro)", "Serbian (Serbia)", "Slovak", "Slovak (Slovakia)", "Slovenian", "Slovenian (Slovenia)", "Spanish", "Spanish (Argentina)", "Spanish (Bolivia)", "Spanish (Chile)", "Spanish (Colombia)", "Spanish (Costa Rica)", "Spanish (Cuba)", "Spanish (Dominican Republic)", "Spanish (Ecuador)", "Spanish (El Salvador)", "Spanish (Guatemala)", "Spanish (Honduras)", "Spanish (Mexico)", "Spanish (Nicaragua)", "Spanish (Panama)", "Spanish (Paraguay)", "Spanish (Peru)", "Spanish (Puerto Rico)", "Spanish (Spain)", "Spanish (United States)", "Spanish (Uruguay)", "Spanish (Venezuela)", "Swedish", "Swedish (Sweden)", "Thai", "Thai (Thailand)", "Thai (Thailand,TH)", "Turkish", "Turkish (Turkey)", "Ukrainian", "Ukrainian (Ukraine)", "Vietnamese", "Vietnamese (Vietnam)" ],
"values" : [ "sq", "sq,AL", "ar", "ar,DZ", "ar,BH", "ar,EG", "ar,IQ", "ar,JO", "ar,KW", "ar,LB", "ar,LY", "ar,MA", "ar,OM", "ar,QA", "ar,SA", "ar,SD", "ar,SY", "ar,TN", "ar,AE", "ar,YE", "be", "be,BY", "bg", "bg,BG", "ca", "ca,ES", "zh", "zh,CN", "zh,HK", "zh,SG", "zh,TW", "hr", "hr,HR", "cs", "cs,CZ", "da", "da,DK", "nl", "nl,BE", "nl,NL", "en", "en,AU", "en,CA", "en,IN", "en,IE", "en,MT", "en,NZ", "en,PH", "en,SG", "en,ZA", "en,GB", "en,US", "et", "et,EE", "fi", "fi,FI", "fr", "fr,BE", "fr,CA", "fr,FR", "fr,LU", "fr,CH", "de", "de,AT", "de,DE", "de,GR", "de,LU", "de,CH", "el", "el,CY", "el,GR", "iw", "iw,IL", "hi", "hi,IN", "hu", "hu,HU", "is", "is,IS", "in", "in,ID", "ga", "ga,IE", "it", "it,IT", "it,CH", "ja", "ja,JP", "ja,JP,JP", "ko", "ko,KR", "lv", "lv,LV", "lt", "lt,LT", "mk", "mk,MK", "ms", "ms,MY", "mt", "mt,MT", "no", "no,NO", "no,NO,NY", "pl", "pl,PL", "pt", "pt,BR", "pt,PT", "ro", "ro,RO", "ru", "ru,RU", "sr", "sr,BA", "sr,ME", "sr,CS", "sr,RS", "sk", "sk,SK", "sl", "sl,SI", "es", "es,AR", "es,BO", "es,CL", "es,CO", "es,CR", "es,CU", "es,DO", "es,EC", "es,SV", "es,GT", "es,HN", "es,MX", "es,NI", "es,PA", "es,PY", "es,PE", "es,PR", "es,ES", "es,US", "es,UY", "es,VE", "sv", "sv,SE", "th", "th,TH", "th,TH,TH", "tr", "tr,TR", "uk", "uk,UA", "vi", "vi,VN" ],
"configDefinitions" : null,
"modelType" : "VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.pipeline.config.LocaleChooserValues",
"filteringConfig" : ""
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 30,
"displayMode" : "BASIC",
"name" : "dataLocale",
"type" : "MODEL",
"defaultValue" : "en,US"
}, {
"fieldName" : "scale",
"description" : "Decimal Value Scale",
"max" : 9223372036854775807,
"label" : "Scale",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : {
"targetType" : [ "DECIMAL" ]
},
"dependsOn" : "",
"group" : "TYPE_CONVERSION",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 40,
"displayMode" : "BASIC",
"name" : "scale",
"type" : "NUMBER",
"defaultValue" : -1
}, {
"fieldName" : "decimalScaleRoundingStrategy",
"description" : "Rounding strategy during scale conversion",
"max" : 9223372036854775807,
"label" : "Rounding Strategy",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : {
"targetType" : [ "DECIMAL" ]
},
"dependsOn" : "",
"group" : "TYPE_CONVERSION",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "Round up", "Round Down", "Round Ceil", "Round Floor", "Round Half Up", "Round Half Down", "Round Half Even", "Round Unnecessary" ],
"values" : [ "ROUND_UP", "ROUND_DOWN", "ROUND_CEILING", "ROUND_FLOOR", "ROUND_HALF_UP", "ROUND_HALF_DOWN", "ROUND_HALF_EVEN", "ROUND_UNNECESSARY" ],
"configDefinitions" : null,
"modelType" : "VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.pipeline.config.DecimalScaleRoundingStrategyChooserValues",
"filteringConfig" : ""
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 50,
"displayMode" : "BASIC",
"name" : "decimalScaleRoundingStrategy",
"type" : "MODEL",
"defaultValue" : "ROUND_UNNECESSARY"
}, {
"fieldName" : "dateFormat",
"description" : "Select or enter any valid date or datetime format",
"max" : 9223372036854775807,
"label" : "Date Format",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"targetType" : [ "DATETIME", "DATE", "STRING", "TIME" ]
},
"dependsOn" : "",
"group" : "TYPE_CONVERSION",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "yyyy-MM-dd", "dd-MMM-yyyy", "yyyy-MM-dd HH:mm:ss", "yyyy-MM-dd HH:mm:ss.SSS", "yyyy-MM-dd HH:mm:ss.SSS Z", "yyyy-MM-dd'T'HH:mm'Z'", "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", "Other ..." ],
"values" : [ "YYYY_MM_DD", "DD_MM_YYYY", "YYYY_MM_DD_HH_MM_SS", "YYYY_MM_DD_HH_MM_SS_SSS", "YYYY_MM_DD_HH_MM_SS_SSS_Z", "YYYY_MM_DD_T_HH_MM_Z", "YYYY_MM_DD_T_HH_MM_SS_SSS_Z", "OTHER" ],
"configDefinitions" : null,
"modelType" : "VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.pipeline.config.DateFormatChooserValues",
"filteringConfig" : ""
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 40,
"displayMode" : "BASIC",
"name" : "dateFormat",
"type" : "MODEL",
"defaultValue" : "YYYY_MM_DD"
}, {
"fieldName" : "otherDateFormat",
"description" : "",
"max" : 9223372036854775807,
"label" : "Other Date Format",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dateFormat" : [ "OTHER" ],
"targetType" : [ "DATETIME", "DATE", "STRING", "TIME" ]
},
"dependsOn" : "",
"group" : "TYPE_CONVERSION",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 50,
"displayMode" : "BASIC",
"name" : "otherDateFormat",
"type" : "STRING",
"defaultValue" : null
}, {
"fieldName" : "zonedDateTimeFormat",
"description" : "Select or enter any valid date or datetime format",
"max" : 9223372036854775807,
"label" : "Zoned DateTime Format",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"targetType" : [ "ZONED_DATETIME", "STRING" ]
},
"dependsOn" : "",
"group" : "TYPE_CONVERSION",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "yyyy-MM-dd'T'HH:mm:ssX", "yyyy-MM-dd'T'HH:mm:ssX[VV]", "Other ..." ],
"values" : [ "ISO_OFFSET_DATE_TIME", "ISO_ZONED_DATE_TIME", "OTHER" ],
"configDefinitions" : null,
"modelType" : "VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.pipeline.ZonedDateTimeFormatChooserValues",
"filteringConfig" : ""
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 60,
"displayMode" : "BASIC",
"name" : "zonedDateTimeFormat",
"type" : "MODEL",
"defaultValue" : "ISO_ZONED_DATE_TIME"
}, {
"fieldName" : "zonedDateTimeTargetTimeZone",
"description" : "Time Zone ",
"max" : 9223372036854775807,
"label" : "Target Time Zone",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"targetType" : [ "ZONED_DATETIME" ]
},
"dependsOn" : "",
"group" : "TYPE_CONVERSION",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "Same as Data Collector: +00:00 UTC (UTC)", "-11:00 ST (US/Samoa)", "-11:00 ST (Pacific/Samoa)", "-11:00 ST (Pacific/Pago_Pago)", "-11:00 ST (Pacific/Midway)", "-11:00 NUT (Pacific/Niue)", "-10:00 TAHT (Pacific/Tahiti)", "-10:00 HT (US/Hawaii)", "-10:00 HT (SystemV/HST10)", "-10:00 HT (Pacific/Johnston)", "-10:00 HT (Pacific/Honolulu)", "-10:00 CKT (Pacific/Rarotonga)", "-09:30 MART (Pacific/Marquesas)", "-09:00 HT (US/Aleutian)", "-09:00 HT (America/Atka)", "-09:00 HT (America/Adak)", "-09:00 GAMT (Pacific/Gambier)", "-09:00 AKT (SystemV/YST9YDT)", "-09:00 AKT (SystemV/YST9)", "-08:00 PT (SystemV/PST8PDT)", "-08:00 PT (SystemV/PST8)", "-08:00 PT (Pacific/Pitcairn)", "-08:00 AKT (US/Alaska)", "-08:00 AKT (America/Yakutat)", "-08:00 AKT (America/Sitka)", "-08:00 AKT (America/Nome)", "-08:00 AKT (America/Metlakatla)", "-08:00 AKT (America/Juneau)", "-08:00 AKT (America/Anchorage)", "-07:00 PT (US/Pacific-New)", "-07:00 PT (US/Pacific)", "-07:00 PT (PST8PDT)", "-07:00 PT (Mexico/BajaNorte)", "-07:00 PT (Canada/Yukon)", "-07:00 PT (Canada/Pacific)", "-07:00 PT (America/Whitehorse)", "-07:00 PT (America/Vancouver)", "-07:00 PT (America/Tijuana)", "-07:00 PT (America/Santa_Isabel)", "-07:00 PT (America/Los_Angeles)", "-07:00 PT (America/Ensenada)", "-07:00 PT (America/Dawson)", "-07:00 MT (US/Arizona)", "-07:00 MT (SystemV/MST7MDT)", "-07:00 MT (SystemV/MST7)", "-07:00 MT (America/Phoenix)", "-07:00 MT (America/Hermosillo)", "-07:00 MT (America/Fort_Nelson)", "-07:00 MT (America/Dawson_Creek)", "-07:00 MT (America/Creston)", "-06:00 MT (US/Mountain)", "-06:00 MT (Navajo)", "-06:00 MT (Mexico/BajaSur)", "-06:00 MT (MST7MDT)", "-06:00 MT (Canada/Mountain)", "-06:00 MT (America/Yellowknife)", "-06:00 MT (America/Shiprock)", "-06:00 MT (America/Ojinaga)", "-06:00 MT (America/Mazatlan)", "-06:00 MT (America/Inuvik)", "-06:00 MT (America/Edmonton)", "-06:00 MT (America/Denver)", "-06:00 MT (America/Chihuahua)", "-06:00 MT (America/Cambridge_Bay)", "-06:00 MT (America/Boise)", "-06:00 GALT (Pacific/Galapagos)", "-06:00 EAST (Pacific/Easter)", "-06:00 EAST (Chile/EasterIsland)", "-06:00 CT (SystemV/CST6CDT)", "-06:00 CT (SystemV/CST6)", "-06:00 CT (Canada/Saskatchewan)", "-06:00 CT (America/Tegucigalpa)", "-06:00 CT (America/Swift_Current)", "-06:00 CT (America/Regina)", "-06:00 CT (America/Managua)", "-06:00 CT (America/Guatemala)", "-06:00 CT (America/El_Salvador)", "-06:00 CT (America/Costa_Rica)", "-06:00 CT (America/Belize)", "-05:00 PET (America/Lima)", "-05:00 ET (SystemV/EST5EDT)", "-05:00 ET (SystemV/EST5)", "-05:00 ET (Jamaica)", "-05:00 ET (America/Panama)", "-05:00 ET (America/Jamaica)", "-05:00 ET (America/Coral_Harbour)", "-05:00 ET (America/Cayman)", "-05:00 ET (America/Cancun)", "-05:00 ET (America/Atikokan)", "-05:00 ECT (America/Guayaquil)", "-05:00 CT (US/Indiana-Starke)", "-05:00 CT (US/Central)", "-05:00 CT (Mexico/General)", "-05:00 CT (Canada/Central)", "-05:00 CT (CST6CDT)", "-05:00 CT (America/Winnipeg)", "-05:00 CT (America/Resolute)", "-05:00 CT (America/Rankin_Inlet)", "-05:00 CT (America/Rainy_River)", "-05:00 CT (America/North_Dakota/New_Salem)", "-05:00 CT (America/North_Dakota/Center)", "-05:00 CT (America/North_Dakota/Beulah)", "-05:00 CT (America/Monterrey)", "-05:00 CT (America/Mexico_City)", "-05:00 CT (America/Merida)", "-05:00 CT (America/Menominee)", "-05:00 CT (America/Matamoros)", "-05:00 CT (America/Knox_IN)", "-05:00 CT (America/Indiana/Tell_City)", "-05:00 CT (America/Indiana/Knox)", "-05:00 CT (America/Chicago)", "-05:00 CT (America/Bahia_Banderas)", "-05:00 COT (America/Bogota)", "-05:00 ACT (Brazil/Acre)", "-05:00 ACT (America/Rio_Branco)", "-05:00 ACT (America/Porto_Acre)", "-05:00 ACT (America/Eirunepe)", "-04:00 VET (America/Caracas)", "-04:00 PYT (America/Asuncion)", "-04:00 GYT (America/Guyana)", "-04:00 ET (US/Michigan)", "-04:00 ET (US/Eastern)", "-04:00 ET (US/East-Indiana)", "-04:00 ET (EST5EDT)", "-04:00 ET (Canada/Eastern)", "-04:00 ET (America/Toronto)", "-04:00 ET (America/Thunder_Bay)", "-04:00 ET (America/Port-au-Prince)", "-04:00 ET (America/Pangnirtung)", "-04:00 ET (America/Nipigon)", "-04:00 ET (America/New_York)", "-04:00 ET (America/Nassau)", "-04:00 ET (America/Montreal)", "-04:00 ET (America/Louisville)", "-04:00 ET (America/Kentucky/Monticello)", "-04:00 ET (America/Kentucky/Louisville)", "-04:00 ET (America/Iqaluit)", "-04:00 ET (America/Indianapolis)", "-04:00 ET (America/Indiana/Winamac)", "-04:00 ET (America/Indiana/Vincennes)", "-04:00 ET (America/Indiana/Vevay)", "-04:00 ET (America/Indiana/Petersburg)", "-04:00 ET (America/Indiana/Marengo)", "-04:00 ET (America/Indiana/Indianapolis)", "-04:00 ET (America/Grand_Turk)", "-04:00 ET (America/Fort_Wayne)", "-04:00 ET (America/Detroit)", "-04:00 CT (Cuba)", "-04:00 CT (America/Havana)", "-04:00 CLT (Chile/Continental)", "-04:00 CLT (America/Santiago)", "-04:00 BOT (America/La_Paz)", "-04:00 AT (SystemV/AST4ADT)", "-04:00 AT (SystemV/AST4)", "-04:00 AT (America/Virgin)", "-04:00 AT (America/Tortola)", "-04:00 AT (America/St_Vincent)", "-04:00 AT (America/St_Thomas)", "-04:00 AT (America/St_Lucia)", "-04:00 AT (America/St_Kitts)", "-04:00 AT (America/St_Barthelemy)", "-04:00 AT (America/Santo_Domingo)", "-04:00 AT (America/Puerto_Rico)", "-04:00 AT (America/Port_of_Spain)", "-04:00 AT (America/Montserrat)", "-04:00 AT (America/Martinique)", "-04:00 AT (America/Marigot)", "-04:00 AT (America/Lower_Princes)", "-04:00 AT (America/Kralendijk)", "-04:00 AT (America/Guadeloupe)", "-04:00 AT (America/Grenada)", "-04:00 AT (America/Dominica)", "-04:00 AT (America/Curacao)", "-04:00 AT (America/Blanc-Sablon)", "-04:00 AT (America/Barbados)", "-04:00 AT (America/Aruba)", "-04:00 AT (America/Antigua)", "-04:00 AT (America/Anguilla)", "-04:00 AMT (Brazil/West)", "-04:00 AMT (America/Porto_Velho)", "-04:00 AMT (America/Manaus)", "-04:00 AMT (America/Cuiaba)", "-04:00 AMT (America/Campo_Grande)", "-04:00 AMT (America/Boa_Vista)", "-03:00 UYT (America/Montevideo)", "-03:00 SRT (America/Paramaribo)", "-03:00 ROTT (Antarctica/Rothera)", "-03:00 GFT (America/Cayenne)", "-03:00 FKT (Atlantic/Stanley)", "-03:00 CLT (Antarctica/Palmer)", "-03:00 BRT (Brazil/East)", "-03:00 BRT (America/Sao_Paulo)", "-03:00 BRT (America/Santarem)", "-03:00 BRT (America/Recife)", "-03:00 BRT (America/Maceio)", "-03:00 BRT (America/Fortaleza)", "-03:00 BRT (America/Belem)", "-03:00 BRT (America/Bahia)", "-03:00 BRT (America/Araguaina)", "-03:00 America/Punta_Arenas (America/Punta_Arenas)", "-03:00 AT (Canada/Atlantic)", "-03:00 AT (Atlantic/Bermuda)", "-03:00 AT (America/Thule)", "-03:00 AT (America/Moncton)", "-03:00 AT (America/Halifax)", "-03:00 AT (America/Goose_Bay)", "-03:00 AT (America/Glace_Bay)", "-03:00 ART (America/Rosario)", "-03:00 ART (America/Mendoza)", "-03:00 ART (America/Jujuy)", "-03:00 ART (America/Cordoba)", "-03:00 ART (America/Catamarca)", "-03:00 ART (America/Buenos_Aires)", "-03:00 ART (America/Argentina/Ushuaia)", "-03:00 ART (America/Argentina/Tucuman)", "-03:00 ART (America/Argentina/San_Luis)", "-03:00 ART (America/Argentina/San_Juan)", "-03:00 ART (America/Argentina/Salta)", "-03:00 ART (America/Argentina/Rio_Gallegos)", "-03:00 ART (America/Argentina/Mendoza)", "-03:00 ART (America/Argentina/La_Rioja)", "-03:00 ART (America/Argentina/Jujuy)", "-03:00 ART (America/Argentina/Cordoba)", "-03:00 ART (America/Argentina/ComodRivadavia)", "-03:00 ART (America/Argentina/Catamarca)", "-03:00 ART (America/Argentina/Buenos_Aires)", "-02:30 NT (Canada/Newfoundland)", "-02:30 NT (America/St_Johns)", "-02:00 WGT (America/Godthab)", "-02:00 PMT (America/Miquelon)", "-02:00 GT (Atlantic/South_Georgia)", "-02:00 FNT (Brazil/DeNoronha)", "-02:00 FNT (America/Noronha)", "-01:00 CVT (Atlantic/Cape_Verde)", "+00:00 AZOT (Atlantic/Azores)", "+00:00 EGT (America/Scoresbysund)", "+00:00 GMT (Africa/Abidjan)", "+00:00 GMT (Africa/Accra)", "+00:00 GMT (Africa/Bamako)", "+00:00 GMT (Africa/Banjul)", "+00:00 GMT (Africa/Bissau)", "+00:00 GMT (Africa/Conakry)", "+00:00 GMT (Africa/Dakar)", "+00:00 GMT (Africa/Freetown)", "+00:00 GMT (Africa/Lome)", "+00:00 GMT (Africa/Monrovia)", "+00:00 GMT (Africa/Nouakchott)", "+00:00 GMT (Africa/Ouagadougou)", "+00:00 GMT (Africa/Sao_Tome)", "+00:00 GMT (Africa/Timbuktu)", "+00:00 GMT (America/Danmarkshavn)", "+00:00 GMT (Atlantic/Reykjavik)", "+00:00 GMT (Atlantic/St_Helena)", "+00:00 GMT (Etc/Greenwich)", "+00:00 GMT (GMT)", "+00:00 GMT (GMT0)", "+00:00 GMT (Greenwich)", "+00:00 GMT (Iceland)", "+00:00 UTC (Etc/UCT)", "+00:00 UTC (Etc/UTC)", "+00:00 UTC (Etc/Universal)", "+00:00 UTC (Etc/Zulu)", "+00:00 UTC (UCT)", "+00:00 UTC (Universal)", "+00:00 UTC (Zulu)", "+01:00 BT (Europe/Belfast)", "+01:00 BT (Europe/Guernsey)", "+01:00 BT (Europe/Isle_of_Man)", "+01:00 BT (Europe/Jersey)", "+01:00 BT (Europe/London)", "+01:00 BT (GB)", "+01:00 BT (GB-Eire)", "+01:00 CET (Africa/Algiers)", "+01:00 CET (Africa/Tunis)", "+01:00 IT (Eire)", "+01:00 IT (Europe/Dublin)", "+01:00 WAT (Africa/Bangui)", "+01:00 WAT (Africa/Brazzaville)", "+01:00 WAT (Africa/Douala)", "+01:00 WAT (Africa/Kinshasa)", "+01:00 WAT (Africa/Lagos)", "+01:00 WAT (Africa/Libreville)", "+01:00 WAT (Africa/Luanda)", "+01:00 WAT (Africa/Malabo)", "+01:00 WAT (Africa/Ndjamena)", "+01:00 WAT (Africa/Niamey)", "+01:00 WAT (Africa/Porto-Novo)", "+01:00 WET (Africa/Casablanca)", "+01:00 WET (Africa/El_Aaiun)", "+01:00 WET (Atlantic/Canary)", "+01:00 WET (Atlantic/Faeroe)", "+01:00 WET (Atlantic/Faroe)", "+01:00 WET (Atlantic/Madeira)", "+01:00 WET (Europe/Lisbon)", "+01:00 WET (Portugal)", "+01:00 WET (WET)", "+02:00 ATT (Antarctica/Troll)", "+02:00 CAT (Africa/Blantyre)", "+02:00 CAT (Africa/Bujumbura)", "+02:00 CAT (Africa/Gaborone)", "+02:00 CAT (Africa/Harare)", "+02:00 CAT (Africa/Khartoum)", "+02:00 CAT (Africa/Kigali)", "+02:00 CAT (Africa/Lubumbashi)", "+02:00 CAT (Africa/Lusaka)", "+02:00 CAT (Africa/Maputo)", "+02:00 CAT (Africa/Windhoek)", "+02:00 CET (Africa/Ceuta)", "+02:00 CET (Arctic/Longyearbyen)", "+02:00 CET (Atlantic/Jan_Mayen)", "+02:00 CET (CET)", "+02:00 CET (Europe/Amsterdam)", "+02:00 CET (Europe/Andorra)", "+02:00 CET (Europe/Belgrade)", "+02:00 CET (Europe/Berlin)", "+02:00 CET (Europe/Bratislava)", "+02:00 CET (Europe/Brussels)", "+02:00 CET (Europe/Budapest)", "+02:00 CET (Europe/Busingen)", "+02:00 CET (Europe/Copenhagen)", "+02:00 CET (Europe/Gibraltar)", "+02:00 CET (Europe/Ljubljana)", "+02:00 CET (Europe/Luxembourg)", "+02:00 CET (Europe/Madrid)", "+02:00 CET (Europe/Malta)", "+02:00 CET (Europe/Monaco)", "+02:00 CET (Europe/Oslo)", "+02:00 CET (Europe/Paris)", "+02:00 CET (Europe/Podgorica)", "+02:00 CET (Europe/Prague)", "+02:00 CET (Europe/Rome)", "+02:00 CET (Europe/San_Marino)", "+02:00 CET (Europe/Sarajevo)", "+02:00 CET (Europe/Skopje)", "+02:00 CET (Europe/Stockholm)", "+02:00 CET (Europe/Tirane)", "+02:00 CET (Europe/Vaduz)", "+02:00 CET (Europe/Vatican)", "+02:00 CET (Europe/Vienna)", "+02:00 CET (Europe/Warsaw)", "+02:00 CET (Europe/Zagreb)", "+02:00 CET (Europe/Zurich)", "+02:00 CET (Poland)", "+02:00 EET (Africa/Cairo)", "+02:00 EET (Africa/Tripoli)", "+02:00 EET (Egypt)", "+02:00 EET (Europe/Kaliningrad)", "+02:00 EET (Libya)", "+02:00 MET (MET)", "+02:00 SAT (Africa/Johannesburg)", "+02:00 SAT (Africa/Maseru)", "+02:00 SAT (Africa/Mbabane)", "+03:00 AT (Asia/Aden)", "+03:00 AT (Asia/Baghdad)", "+03:00 AT (Asia/Bahrain)", "+03:00 AT (Asia/Kuwait)", "+03:00 AT (Asia/Qatar)", "+03:00 AT (Asia/Riyadh)", "+03:00 Asia/Famagusta (Asia/Famagusta)", "+03:00 EAT (Africa/Addis_Ababa)", "+03:00 EAT (Africa/Asmara)", "+03:00 EAT (Africa/Asmera)", "+03:00 EAT (Africa/Dar_es_Salaam)", "+03:00 EAT (Africa/Djibouti)", "+03:00 EAT (Africa/Juba)", "+03:00 EAT (Africa/Kampala)", "+03:00 EAT (Africa/Mogadishu)", "+03:00 EAT (Africa/Nairobi)", "+03:00 EAT (Indian/Antananarivo)", "+03:00 EAT (Indian/Comoro)", "+03:00 EAT (Indian/Mayotte)", "+03:00 EET (Asia/Amman)", "+03:00 EET (Asia/Beirut)", "+03:00 EET (Asia/Damascus)", "+03:00 EET (Asia/Gaza)", "+03:00 EET (Asia/Hebron)", "+03:00 EET (Asia/Istanbul)", "+03:00 EET (Asia/Nicosia)", "+03:00 EET (EET)", "+03:00 EET (Europe/Athens)", "+03:00 EET (Europe/Bucharest)", "+03:00 EET (Europe/Chisinau)", "+03:00 EET (Europe/Helsinki)", "+03:00 EET (Europe/Istanbul)", "+03:00 EET (Europe/Kiev)", "+03:00 EET (Europe/Mariehamn)", "+03:00 EET (Europe/Nicosia)", "+03:00 EET (Europe/Riga)", "+03:00 EET (Europe/Sofia)", "+03:00 EET (Europe/Tallinn)", "+03:00 EET (Europe/Tiraspol)", "+03:00 EET (Europe/Uzhgorod)", "+03:00 EET (Europe/Vilnius)", "+03:00 EET (Europe/Zaporozhye)", "+03:00 EET (Turkey)", "+03:00 Europe/Kirov (Europe/Kirov)", "+03:00 IT (Asia/Jerusalem)", "+03:00 IT (Asia/Tel_Aviv)", "+03:00 IT (Israel)", "+03:00 MT (Europe/Minsk)", "+03:00 MT (Europe/Moscow)", "+03:00 MT (Europe/Simferopol)", "+03:00 MT (W-SU)", "+03:00 SYOT (Antarctica/Syowa)", "+04:00 AMT (Asia/Yerevan)", "+04:00 AZT (Asia/Baku)", "+04:00 Europe/Astrakhan (Europe/Astrakhan)", "+04:00 Europe/Saratov (Europe/Saratov)", "+04:00 Europe/Ulyanovsk (Europe/Ulyanovsk)", "+04:00 GET (Asia/Tbilisi)", "+04:00 GT (Asia/Dubai)", "+04:00 GT (Asia/Muscat)", "+04:00 MT (Europe/Volgograd)", "+04:00 MUT (Indian/Mauritius)", "+04:00 RET (Indian/Reunion)", "+04:00 SAMT (Europe/Samara)", "+04:00 SCT (Indian/Mahe)", "+04:30 AFT (Asia/Kabul)", "+04:30 IRT (Asia/Tehran)", "+04:30 IRT (Iran)", "+05:00 AQTT (Asia/Aqtau)", "+05:00 AQTT (Asia/Aqtobe)", "+05:00 Asia/Atyrau (Asia/Atyrau)", "+05:00 MAWT (Antarctica/Mawson)", "+05:00 MVT (Indian/Maldives)", "+05:00 ORAT (Asia/Oral)", "+05:00 PKT (Asia/Karachi)", "+05:00 QYZT (Asia/Qyzylorda)", "+05:00 TFT (Indian/Kerguelen)", "+05:00 TJT (Asia/Dushanbe)", "+05:00 TMT (Asia/Ashgabat)", "+05:00 TMT (Asia/Ashkhabad)", "+05:00 UZT (Asia/Samarkand)", "+05:00 UZT (Asia/Tashkent)", "+05:00 YEKT (Asia/Yekaterinburg)", "+05:30 IT (Asia/Calcutta)", "+05:30 IT (Asia/Colombo)", "+05:30 IT (Asia/Kolkata)", "+05:45 NPT (Asia/Kathmandu)", "+05:45 NPT (Asia/Katmandu)", "+06:00 ALMT (Asia/Almaty)", "+06:00 BDT (Asia/Dacca)", "+06:00 BDT (Asia/Dhaka)", "+06:00 BTT (Asia/Thimbu)", "+06:00 BTT (Asia/Thimphu)", "+06:00 IOT (Indian/Chagos)", "+06:00 KGT (Asia/Bishkek)", "+06:00 OMST (Asia/Omsk)", "+06:00 QOST (Asia/Qostanay)", "+06:00 VOST (Antarctica/Vostok)", "+06:00 XJT (Asia/Kashgar)", "+06:00 XJT (Asia/Urumqi)", "+06:30 CCT (Indian/Cocos)", "+06:30 MMT (Asia/Rangoon)", "+06:30 MMT (Asia/Yangon)", "+07:00 Asia/Barnaul (Asia/Barnaul)", "+07:00 Asia/Tomsk (Asia/Tomsk)", "+07:00 CIT (Indian/Christmas)", "+07:00 DAVT (Antarctica/Davis)", "+07:00 HOVT (Asia/Hovd)", "+07:00 ICT (Asia/Bangkok)", "+07:00 ICT (Asia/Ho_Chi_Minh)", "+07:00 ICT (Asia/Phnom_Penh)", "+07:00 ICT (Asia/Saigon)", "+07:00 ICT (Asia/Vientiane)", "+07:00 KRAT (Asia/Krasnoyarsk)", "+07:00 KRAT (Asia/Novokuznetsk)", "+07:00 NOVT (Asia/Novosibirsk)", "+07:00 WIB (Asia/Jakarta)", "+07:00 WIB (Asia/Pontianak)", "+08:00 AWT (Antarctica/Casey)", "+08:00 AWT (Australia/Perth)", "+08:00 AWT (Australia/West)", "+08:00 BNT (Asia/Brunei)", "+08:00 CHOT (Asia/Choibalsan)", "+08:00 CT (Asia/Chongqing)", "+08:00 CT (Asia/Chungking)", "+08:00 CT (Asia/Harbin)", "+08:00 CT (Asia/Macao)", "+08:00 CT (Asia/Macau)", "+08:00 CT (Asia/Shanghai)", "+08:00 CT (Asia/Taipei)", "+08:00 CT (PRC)", "+08:00 HKT (Asia/Hong_Kong)", "+08:00 HKT (Hongkong)", "+08:00 IRKT (Asia/Irkutsk)", "+08:00 MYT (Asia/Kuala_Lumpur)", "+08:00 MYT (Asia/Kuching)", "+08:00 PT (Asia/Manila)", "+08:00 SGT (Asia/Singapore)", "+08:00 SGT (Singapore)", "+08:00 ULAT (Asia/Ulaanbaatar)", "+08:00 ULAT (Asia/Ulan_Bator)", "+08:00 WITA (Asia/Makassar)", "+08:00 WITA (Asia/Ujung_Pandang)", "+08:45 ACWT (Australia/Eucla)", "+09:00 JT (Asia/Tokyo)", "+09:00 JT (Japan)", "+09:00 KT (Asia/Pyongyang)", "+09:00 KT (Asia/Seoul)", "+09:00 KT (ROK)", "+09:00 PWT (Pacific/Palau)", "+09:00 TLT (Asia/Dili)", "+09:00 WIT (Asia/Jayapura)", "+09:00 YAKT (Asia/Chita)", "+09:00 YAKT (Asia/Khandyga)", "+09:00 YAKT (Asia/Yakutsk)", "+09:30 ACT (Australia/Adelaide)", "+09:30 ACT (Australia/Broken_Hill)", "+09:30 ACT (Australia/Darwin)", "+09:30 ACT (Australia/North)", "+09:30 ACT (Australia/South)", "+09:30 ACT (Australia/Yancowinna)", "+10:00 AET (Australia/ACT)", "+10:00 AET (Australia/Brisbane)", "+10:00 AET (Australia/Canberra)", "+10:00 AET (Australia/Currie)", "+10:00 AET (Australia/Hobart)", "+10:00 AET (Australia/Lindeman)", "+10:00 AET (Australia/Melbourne)", "+10:00 AET (Australia/NSW)", "+10:00 AET (Australia/Queensland)", "+10:00 AET (Australia/Sydney)", "+10:00 AET (Australia/Tasmania)", "+10:00 AET (Australia/Victoria)", "+10:00 CHUT (Pacific/Chuuk)", "+10:00 CHUT (Pacific/Truk)", "+10:00 CHUT (Pacific/Yap)", "+10:00 ChT (Pacific/Guam)", "+10:00 ChT (Pacific/Saipan)", "+10:00 DDUT (Antarctica/DumontDUrville)", "+10:00 PGT (Pacific/Port_Moresby)", "+10:00 VLAT (Asia/Ust-Nera)", "+10:00 VLAT (Asia/Vladivostok)", "+10:30 LHT (Australia/LHI)", "+10:30 LHT (Australia/Lord_Howe)", "+11:00 BT (Pacific/Bougainville)", "+11:00 KOST (Pacific/Kosrae)", "+11:00 MAGT (Asia/Magadan)", "+11:00 MIST (Antarctica/Macquarie)", "+11:00 NCT (Pacific/Noumea)", "+11:00 NFT (Pacific/Norfolk)", "+11:00 PONT (Pacific/Pohnpei)", "+11:00 PONT (Pacific/Ponape)", "+11:00 SAKT (Asia/Sakhalin)", "+11:00 SBT (Pacific/Guadalcanal)", "+11:00 SRET (Asia/Srednekolymsk)", "+11:00 VUT (Pacific/Efate)", "+12:00 ANAT (Asia/Anadyr)", "+12:00 FJT (Pacific/Fiji)", "+12:00 GILT (Pacific/Tarawa)", "+12:00 MHT (Kwajalein)", "+12:00 MHT (Pacific/Kwajalein)", "+12:00 MHT (Pacific/Majuro)", "+12:00 NRT (Pacific/Nauru)", "+12:00 NZT (Antarctica/McMurdo)", "+12:00 NZT (Antarctica/South_Pole)", "+12:00 NZT (NZ)", "+12:00 NZT (Pacific/Auckland)", "+12:00 PETT (Asia/Kamchatka)", "+12:00 TVT (Pacific/Funafuti)", "+12:00 WAKT (Pacific/Wake)", "+12:00 WFT (Pacific/Wallis)", "+12:45 CHAT (NZ-CHAT)", "+12:45 CHAT (Pacific/Chatham)", "+13:00 PHOT (Pacific/Enderbury)", "+13:00 TKT (Pacific/Fakaofo)", "+13:00 TOT (Pacific/Tongatapu)", "+13:00 WST (Pacific/Apia)", "+14:00 LINT (Pacific/Kiritimati)" ],
"values" : [ "UTC", "US/Samoa", "Pacific/Samoa", "Pacific/Pago_Pago", "Pacific/Midway", "Pacific/Niue", "Pacific/Tahiti", "US/Hawaii", "SystemV/HST10", "Pacific/Johnston", "Pacific/Honolulu", "Pacific/Rarotonga", "Pacific/Marquesas", "US/Aleutian", "America/Atka", "America/Adak", "Pacific/Gambier", "SystemV/YST9YDT", "SystemV/YST9", "SystemV/PST8PDT", "SystemV/PST8", "Pacific/Pitcairn", "US/Alaska", "America/Yakutat", "America/Sitka", "America/Nome", "America/Metlakatla", "America/Juneau", "America/Anchorage", "US/Pacific-New", "US/Pacific", "PST8PDT", "Mexico/BajaNorte", "Canada/Yukon", "Canada/Pacific", "America/Whitehorse", "America/Vancouver", "America/Tijuana", "America/Santa_Isabel", "America/Los_Angeles", "America/Ensenada", "America/Dawson", "US/Arizona", "SystemV/MST7MDT", "SystemV/MST7", "America/Phoenix", "America/Hermosillo", "America/Fort_Nelson", "America/Dawson_Creek", "America/Creston", "US/Mountain", "Navajo", "Mexico/BajaSur", "MST7MDT", "Canada/Mountain", "America/Yellowknife", "America/Shiprock", "America/Ojinaga", "America/Mazatlan", "America/Inuvik", "America/Edmonton", "America/Denver", "America/Chihuahua", "America/Cambridge_Bay", "America/Boise", "Pacific/Galapagos", "Pacific/Easter", "Chile/EasterIsland", "SystemV/CST6CDT", "SystemV/CST6", "Canada/Saskatchewan", "America/Tegucigalpa", "America/Swift_Current", "America/Regina", "America/Managua", "America/Guatemala", "America/El_Salvador", "America/Costa_Rica", "America/Belize", "America/Lima", "SystemV/EST5EDT", "SystemV/EST5", "Jamaica", "America/Panama", "America/Jamaica", "America/Coral_Harbour", "America/Cayman", "America/Cancun", "America/Atikokan", "America/Guayaquil", "US/Indiana-Starke", "US/Central", "Mexico/General", "Canada/Central", "CST6CDT", "America/Winnipeg", "America/Resolute", "America/Rankin_Inlet", "America/Rainy_River", "America/North_Dakota/New_Salem", "America/North_Dakota/Center", "America/North_Dakota/Beulah", "America/Monterrey", "America/Mexico_City", "America/Merida", "America/Menominee", "America/Matamoros", "America/Knox_IN", "America/Indiana/Tell_City", "America/Indiana/Knox", "America/Chicago", "America/Bahia_Banderas", "America/Bogota", "Brazil/Acre", "America/Rio_Branco", "America/Porto_Acre", "America/Eirunepe", "America/Caracas", "America/Asuncion", "America/Guyana", "US/Michigan", "US/Eastern", "US/East-Indiana", "EST5EDT", "Canada/Eastern", "America/Toronto", "America/Thunder_Bay", "America/Port-au-Prince", "America/Pangnirtung", "America/Nipigon", "America/New_York", "America/Nassau", "America/Montreal", "America/Louisville", "America/Kentucky/Monticello", "America/Kentucky/Louisville", "America/Iqaluit", "America/Indianapolis", "America/Indiana/Winamac", "America/Indiana/Vincennes", "America/Indiana/Vevay", "America/Indiana/Petersburg", "America/Indiana/Marengo", "America/Indiana/Indianapolis", "America/Grand_Turk", "America/Fort_Wayne", "America/Detroit", "Cuba", "America/Havana", "Chile/Continental", "America/Santiago", "America/La_Paz", "SystemV/AST4ADT", "SystemV/AST4", "America/Virgin", "America/Tortola", "America/St_Vincent", "America/St_Thomas", "America/St_Lucia", "America/St_Kitts", "America/St_Barthelemy", "America/Santo_Domingo", "America/Puerto_Rico", "America/Port_of_Spain", "America/Montserrat", "America/Martinique", "America/Marigot", "America/Lower_Princes", "America/Kralendijk", "America/Guadeloupe", "America/Grenada", "America/Dominica", "America/Curacao", "America/Blanc-Sablon", "America/Barbados", "America/Aruba", "America/Antigua", "America/Anguilla", "Brazil/West", "America/Porto_Velho", "America/Manaus", "America/Cuiaba", "America/Campo_Grande", "America/Boa_Vista", "America/Montevideo", "America/Paramaribo", "Antarctica/Rothera", "America/Cayenne", "Atlantic/Stanley", "Antarctica/Palmer", "Brazil/East", "America/Sao_Paulo", "America/Santarem", "America/Recife", "America/Maceio", "America/Fortaleza", "America/Belem", "America/Bahia", "America/Araguaina", "America/Punta_Arenas", "Canada/Atlantic", "Atlantic/Bermuda", "America/Thule", "America/Moncton", "America/Halifax", "America/Goose_Bay", "America/Glace_Bay", "America/Rosario", "America/Mendoza", "America/Jujuy", "America/Cordoba", "America/Catamarca", "America/Buenos_Aires", "America/Argentina/Ushuaia", "America/Argentina/Tucuman", "America/Argentina/San_Luis", "America/Argentina/San_Juan", "America/Argentina/Salta", "America/Argentina/Rio_Gallegos", "America/Argentina/Mendoza", "America/Argentina/La_Rioja", "America/Argentina/Jujuy", "America/Argentina/Cordoba", "America/Argentina/ComodRivadavia", "America/Argentina/Catamarca", "America/Argentina/Buenos_Aires", "Canada/Newfoundland", "America/St_Johns", "America/Godthab", "America/Miquelon", "Atlantic/South_Georgia", "Brazil/DeNoronha", "America/Noronha", "Atlantic/Cape_Verde", "Atlantic/Azores", "America/Scoresbysund", "Africa/Abidjan", "Africa/Accra", "Africa/Bamako", "Africa/Banjul", "Africa/Bissau", "Africa/Conakry", "Africa/Dakar", "Africa/Freetown", "Africa/Lome", "Africa/Monrovia", "Africa/Nouakchott", "Africa/Ouagadougou", "Africa/Sao_Tome", "Africa/Timbuktu", "America/Danmarkshavn", "Atlantic/Reykjavik", "Atlantic/St_Helena", "Etc/Greenwich", "GMT", "GMT0", "Greenwich", "Iceland", "Etc/UCT", "Etc/UTC", "Etc/Universal", "Etc/Zulu", "UCT", "Universal", "Zulu", "Europe/Belfast", "Europe/Guernsey", "Europe/Isle_of_Man", "Europe/Jersey", "Europe/London", "GB", "GB-Eire", "Africa/Algiers", "Africa/Tunis", "Eire", "Europe/Dublin", "Africa/Bangui", "Africa/Brazzaville", "Africa/Douala", "Africa/Kinshasa", "Africa/Lagos", "Africa/Libreville", "Africa/Luanda", "Africa/Malabo", "Africa/Ndjamena", "Africa/Niamey", "Africa/Porto-Novo", "Africa/Casablanca", "Africa/El_Aaiun", "Atlantic/Canary", "Atlantic/Faeroe", "Atlantic/Faroe", "Atlantic/Madeira", "Europe/Lisbon", "Portugal", "WET", "Antarctica/Troll", "Africa/Blantyre", "Africa/Bujumbura", "Africa/Gaborone", "Africa/Harare", "Africa/Khartoum", "Africa/Kigali", "Africa/Lubumbashi", "Africa/Lusaka", "Africa/Maputo", "Africa/Windhoek", "Africa/Ceuta", "Arctic/Longyearbyen", "Atlantic/Jan_Mayen", "CET", "Europe/Amsterdam", "Europe/Andorra", "Europe/Belgrade", "Europe/Berlin", "Europe/Bratislava", "Europe/Brussels", "Europe/Budapest", "Europe/Busingen", "Europe/Copenhagen", "Europe/Gibraltar", "Europe/Ljubljana", "Europe/Luxembourg", "Europe/Madrid", "Europe/Malta", "Europe/Monaco", "Europe/Oslo", "Europe/Paris", "Europe/Podgorica", "Europe/Prague", "Europe/Rome", "Europe/San_Marino", "Europe/Sarajevo", "Europe/Skopje", "Europe/Stockholm", "Europe/Tirane", "Europe/Vaduz", "Europe/Vatican", "Europe/Vienna", "Europe/Warsaw", "Europe/Zagreb", "Europe/Zurich", "Poland", "Africa/Cairo", "Africa/Tripoli", "Egypt", "Europe/Kaliningrad", "Libya", "MET", "Africa/Johannesburg", "Africa/Maseru", "Africa/Mbabane", "Asia/Aden", "Asia/Baghdad", "Asia/Bahrain", "Asia/Kuwait", "Asia/Qatar", "Asia/Riyadh", "Asia/Famagusta", "Africa/Addis_Ababa", "Africa/Asmara", "Africa/Asmera", "Africa/Dar_es_Salaam", "Africa/Djibouti", "Africa/Juba", "Africa/Kampala", "Africa/Mogadishu", "Africa/Nairobi", "Indian/Antananarivo", "Indian/Comoro", "Indian/Mayotte", "Asia/Amman", "Asia/Beirut", "Asia/Damascus", "Asia/Gaza", "Asia/Hebron", "Asia/Istanbul", "Asia/Nicosia", "EET", "Europe/Athens", "Europe/Bucharest", "Europe/Chisinau", "Europe/Helsinki", "Europe/Istanbul", "Europe/Kiev", "Europe/Mariehamn", "Europe/Nicosia", "Europe/Riga", "Europe/Sofia", "Europe/Tallinn", "Europe/Tiraspol", "Europe/Uzhgorod", "Europe/Vilnius", "Europe/Zaporozhye", "Turkey", "Europe/Kirov", "Asia/Jerusalem", "Asia/Tel_Aviv", "Israel", "Europe/Minsk", "Europe/Moscow", "Europe/Simferopol", "W-SU", "Antarctica/Syowa", "Asia/Yerevan", "Asia/Baku", "Europe/Astrakhan", "Europe/Saratov", "Europe/Ulyanovsk", "Asia/Tbilisi", "Asia/Dubai", "Asia/Muscat", "Europe/Volgograd", "Indian/Mauritius", "Indian/Reunion", "Europe/Samara", "Indian/Mahe", "Asia/Kabul", "Asia/Tehran", "Iran", "Asia/Aqtau", "Asia/Aqtobe", "Asia/Atyrau", "Antarctica/Mawson", "Indian/Maldives", "Asia/Oral", "Asia/Karachi", "Asia/Qyzylorda", "Indian/Kerguelen", "Asia/Dushanbe", "Asia/Ashgabat", "Asia/Ashkhabad", "Asia/Samarkand", "Asia/Tashkent", "Asia/Yekaterinburg", "Asia/Calcutta", "Asia/Colombo", "Asia/Kolkata", "Asia/Kathmandu", "Asia/Katmandu", "Asia/Almaty", "Asia/Dacca", "Asia/Dhaka", "Asia/Thimbu", "Asia/Thimphu", "Indian/Chagos", "Asia/Bishkek", "Asia/Omsk", "Asia/Qostanay", "Antarctica/Vostok", "Asia/Kashgar", "Asia/Urumqi", "Indian/Cocos", "Asia/Rangoon", "Asia/Yangon", "Asia/Barnaul", "Asia/Tomsk", "Indian/Christmas", "Antarctica/Davis", "Asia/Hovd", "Asia/Bangkok", "Asia/Ho_Chi_Minh", "Asia/Phnom_Penh", "Asia/Saigon", "Asia/Vientiane", "Asia/Krasnoyarsk", "Asia/Novokuznetsk", "Asia/Novosibirsk", "Asia/Jakarta", "Asia/Pontianak", "Antarctica/Casey", "Australia/Perth", "Australia/West", "Asia/Brunei", "Asia/Choibalsan", "Asia/Chongqing", "Asia/Chungking", "Asia/Harbin", "Asia/Macao", "Asia/Macau", "Asia/Shanghai", "Asia/Taipei", "PRC", "Asia/Hong_Kong", "Hongkong", "Asia/Irkutsk", "Asia/Kuala_Lumpur", "Asia/Kuching", "Asia/Manila", "Asia/Singapore", "Singapore", "Asia/Ulaanbaatar", "Asia/Ulan_Bator", "Asia/Makassar", "Asia/Ujung_Pandang", "Australia/Eucla", "Asia/Tokyo", "Japan", "Asia/Pyongyang", "Asia/Seoul", "ROK", "Pacific/Palau", "Asia/Dili", "Asia/Jayapura", "Asia/Chita", "Asia/Khandyga", "Asia/Yakutsk", "Australia/Adelaide", "Australia/Broken_Hill", "Australia/Darwin", "Australia/North", "Australia/South", "Australia/Yancowinna", "Australia/ACT", "Australia/Brisbane", "Australia/Canberra", "Australia/Currie", "Australia/Hobart", "Australia/Lindeman", "Australia/Melbourne", "Australia/NSW", "Australia/Queensland", "Australia/Sydney", "Australia/Tasmania", "Australia/Victoria", "Pacific/Chuuk", "Pacific/Truk", "Pacific/Yap", "Pacific/Guam", "Pacific/Saipan", "Antarctica/DumontDUrville", "Pacific/Port_Moresby", "Asia/Ust-Nera", "Asia/Vladivostok", "Australia/LHI", "Australia/Lord_Howe", "Pacific/Bougainville", "Pacific/Kosrae", "Asia/Magadan", "Antarctica/Macquarie", "Pacific/Noumea", "Pacific/Norfolk", "Pacific/Pohnpei", "Pacific/Ponape", "Asia/Sakhalin", "Pacific/Guadalcanal", "Asia/Srednekolymsk", "Pacific/Efate", "Asia/Anadyr", "Pacific/Fiji", "Pacific/Tarawa", "Kwajalein", "Pacific/Kwajalein", "Pacific/Majuro", "Pacific/Nauru", "Antarctica/McMurdo", "Antarctica/South_Pole", "NZ", "Pacific/Auckland", "Asia/Kamchatka", "Pacific/Funafuti", "Pacific/Wake", "Pacific/Wallis", "NZ-CHAT", "Pacific/Chatham", "Pacific/Enderbury", "Pacific/Fakaofo", "Pacific/Tongatapu", "Pacific/Apia", "Pacific/Kiritimati" ],
"configDefinitions" : null,
"modelType" : "VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.pipeline.config.TimeZoneChooserValues",
"filteringConfig" : ""
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 180,
"displayMode" : "BASIC",
"name" : "zonedDateTimeTargetTimeZone",
"type" : "MODEL",
"defaultValue" : null
}, {
"fieldName" : "otherZonedDateTimeFormat",
"description" : "",
"max" : 9223372036854775807,
"label" : "Other Zoned DateTime Format",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"zonedDateTimeFormat" : [ "OTHER" ],
"targetType" : [ "ZONED_DATETIME", "STRING" ]
},
"dependsOn" : "",
"group" : "TYPE_CONVERSION",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 70,
"displayMode" : "BASIC",
"name" : "otherZonedDateTimeFormat",
"type" : "STRING",
"defaultValue" : null
}, {
"fieldName" : "encoding",
"description" : "",
"max" : 9223372036854775807,
"label" : "CharSet",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"targetType" : [ "STRING" ]
},
"dependsOn" : "",
"group" : "TYPE_CONVERSION",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "UTF-8", "US-ASCII", "UTF-16", "ISO-8859-1", "IBM-500", "GBK", "Big5", "Big5-HKSCS", "CESU-8", "EUC-JP", "EUC-KR", "GB18030", "GB2312", "IBM-Thai", "IBM00858", "IBM01140", "IBM01141", "IBM01142", "IBM01143", "IBM01144", "IBM01145", "IBM01146", "IBM01147", "IBM01148", "IBM01149", "IBM037", "IBM1026", "IBM1047", "IBM273", "IBM277", "IBM278", "IBM280", "IBM284", "IBM285", "IBM290", "IBM297", "IBM420", "IBM424", "IBM437", "IBM500", "IBM775", "IBM850", "IBM852", "IBM855", "IBM857", "IBM860", "IBM861", "IBM862", "IBM863", "IBM864", "IBM865", "IBM866", "IBM868", "IBM869", "IBM870", "IBM871", "IBM918", "ISO-2022-CN", "ISO-2022-JP", "ISO-2022-JP-2", "ISO-2022-KR", "ISO-8859-13", "ISO-8859-15", "ISO-8859-2", "ISO-8859-3", "ISO-8859-4", "ISO-8859-5", "ISO-8859-6", "ISO-8859-7", "ISO-8859-8", "ISO-8859-9", "JIS_X0201", "JIS_X0212-1990", "KOI8-R", "KOI8-U", "Shift_JIS", "TIS-620", "UTF-16BE", "UTF-16LE", "UTF-32", "UTF-32BE", "UTF-32LE", "windows-1250", "windows-1251", "windows-1252", "windows-1253", "windows-1254", "windows-1255", "windows-1256", "windows-1257", "windows-1258", "windows-31j", "x-Big5-HKSCS-2001", "x-Big5-Solaris", "x-COMPOUND_TEXT", "x-euc-jp-linux", "x-EUC-TW", "x-eucJP-Open", "x-IBM1006", "x-IBM1025", "x-IBM1046", "x-IBM1097", "x-IBM1098", "x-IBM1112", "x-IBM1122", "x-IBM1123", "x-IBM1124", "x-IBM1166", "x-IBM1364", "x-IBM1381", "x-IBM1383", "x-IBM300", "x-IBM33722", "x-IBM737", "x-IBM833", "x-IBM834", "x-IBM856", "x-IBM874", "x-IBM875", "x-IBM921", "x-IBM922", "x-IBM930", "x-IBM933", "x-IBM935", "x-IBM937", "x-IBM939", "x-IBM942", "x-IBM942C", "x-IBM943", "x-IBM943C", "x-IBM948", "x-IBM949", "x-IBM949C", "x-IBM950", "x-IBM964", "x-IBM970", "x-ISCII91", "x-ISO-2022-CN-CNS", "x-ISO-2022-CN-GB", "x-iso-8859-11", "x-JIS0208", "x-JISAutoDetect", "x-Johab", "x-MacArabic", "x-MacCentralEurope", "x-MacCroatian", "x-MacCyrillic", "x-MacDingbat", "x-MacGreek", "x-MacHebrew", "x-MacIceland", "x-MacRoman", "x-MacRomania", "x-MacSymbol", "x-MacThai", "x-MacTurkish", "x-MacUkraine", "x-MS932_0213", "x-MS950-HKSCS", "x-MS950-HKSCS-XP", "x-mswin-936", "x-PCK", "x-SJIS_0213", "x-UTF-16LE-BOM", "X-UTF-32BE-BOM", "X-UTF-32LE-BOM", "x-windows-50220", "x-windows-50221", "x-windows-874", "x-windows-949", "x-windows-950", "x-windows-iso2022jp" ],
"values" : [ "UTF-8", "US-ASCII", "UTF-16", "ISO-8859-1", "IBM-500", "GBK", "Big5", "Big5-HKSCS", "CESU-8", "EUC-JP", "EUC-KR", "GB18030", "GB2312", "IBM-Thai", "IBM00858", "IBM01140", "IBM01141", "IBM01142", "IBM01143", "IBM01144", "IBM01145", "IBM01146", "IBM01147", "IBM01148", "IBM01149", "IBM037", "IBM1026", "IBM1047", "IBM273", "IBM277", "IBM278", "IBM280", "IBM284", "IBM285", "IBM290", "IBM297", "IBM420", "IBM424", "IBM437", "IBM500", "IBM775", "IBM850", "IBM852", "IBM855", "IBM857", "IBM860", "IBM861", "IBM862", "IBM863", "IBM864", "IBM865", "IBM866", "IBM868", "IBM869", "IBM870", "IBM871", "IBM918", "ISO-2022-CN", "ISO-2022-JP", "ISO-2022-JP-2", "ISO-2022-KR", "ISO-8859-13", "ISO-8859-15", "ISO-8859-2", "ISO-8859-3", "ISO-8859-4", "ISO-8859-5", "ISO-8859-6", "ISO-8859-7", "ISO-8859-8", "ISO-8859-9", "JIS_X0201", "JIS_X0212-1990", "KOI8-R", "KOI8-U", "Shift_JIS", "TIS-620", "UTF-16BE", "UTF-16LE", "UTF-32", "UTF-32BE", "UTF-32LE", "windows-1250", "windows-1251", "windows-1252", "windows-1253", "windows-1254", "windows-1255", "windows-1256", "windows-1257", "windows-1258", "windows-31j", "x-Big5-HKSCS-2001", "x-Big5-Solaris", "x-COMPOUND_TEXT", "x-euc-jp-linux", "x-EUC-TW", "x-eucJP-Open", "x-IBM1006", "x-IBM1025", "x-IBM1046", "x-IBM1097", "x-IBM1098", "x-IBM1112", "x-IBM1122", "x-IBM1123", "x-IBM1124", "x-IBM1166", "x-IBM1364", "x-IBM1381", "x-IBM1383", "x-IBM300", "x-IBM33722", "x-IBM737", "x-IBM833", "x-IBM834", "x-IBM856", "x-IBM874", "x-IBM875", "x-IBM921", "x-IBM922", "x-IBM930", "x-IBM933", "x-IBM935", "x-IBM937", "x-IBM939", "x-IBM942", "x-IBM942C", "x-IBM943", "x-IBM943C", "x-IBM948", "x-IBM949", "x-IBM949C", "x-IBM950", "x-IBM964", "x-IBM970", "x-ISCII91", "x-ISO-2022-CN-CNS", "x-ISO-2022-CN-GB", "x-iso-8859-11", "x-JIS0208", "x-JISAutoDetect", "x-Johab", "x-MacArabic", "x-MacCentralEurope", "x-MacCroatian", "x-MacCyrillic", "x-MacDingbat", "x-MacGreek", "x-MacHebrew", "x-MacIceland", "x-MacRoman", "x-MacRomania", "x-MacSymbol", "x-MacThai", "x-MacTurkish", "x-MacUkraine", "x-MS932_0213", "x-MS950-HKSCS", "x-MS950-HKSCS-XP", "x-mswin-936", "x-PCK", "x-SJIS_0213", "x-UTF-16LE-BOM", "X-UTF-32BE-BOM", "X-UTF-32LE-BOM", "x-windows-50220", "x-windows-50221", "x-windows-874", "x-windows-949", "x-windows-950", "x-windows-iso2022jp" ],
"configDefinitions" : null,
"modelType" : "VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.pipeline.config.CharsetChooserValues",
"filteringConfig" : ""
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 80,
"displayMode" : "BASIC",
"name" : "encoding",
"type" : "MODEL",
"defaultValue" : "UTF-8"
} ],
"modelType" : "LIST_BEAN",
"valuesProviderClass" : null,
"filteringConfig" : null
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 10,
"displayMode" : "BASIC",
"name" : "fieldTypeConverterConfigs",
"type" : "MODEL",
"defaultValue" : [ ]
}, {
"fieldName" : "wholeTypeConverterConfigs",
"description" : "Configure types that should be converted. All fields of given type in a record will be converted.",
"max" : 9223372036854775807,
"label" : "",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : {
"convertBy" : [ "BY_TYPE" ]
},
"dependsOn" : "",
"group" : "TYPE_CONVERSION",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : null,
"values" : null,
"configDefinitions" : [ {
"fieldName" : "sourceType",
"description" : "Converts all fields of given type",
"max" : 9223372036854775807,
"label" : "Source type",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : { },
"dependsOn" : "",
"group" : "TYPE_CONVERSION",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "BOOLEAN", "CHAR", "BYTE", "SHORT", "INTEGER", "LONG", "FLOAT", "DOUBLE", "DECIMAL", "DATE", "TIME", "DATETIME", "ZONED_DATETIME", "STRING", "BYTE_ARRAY" ],
"values" : [ "BOOLEAN", "CHAR", "BYTE", "SHORT", "INTEGER", "LONG", "FLOAT", "DOUBLE", "DECIMAL", "DATE", "TIME", "DATETIME", "ZONED_DATETIME", "STRING", "BYTE_ARRAY" ],
"configDefinitions" : null,
"modelType" : "VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.pipeline.config.PrimitiveFieldTypeChooserValues",
"filteringConfig" : ""
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 10,
"displayMode" : "BASIC",
"name" : "sourceType",
"type" : "MODEL",
"defaultValue" : "INTEGER"
}, {
"fieldName" : "targetType",
"description" : "Select a compatible data type",
"max" : 9223372036854775807,
"label" : "Convert to Type",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : { },
"dependsOn" : "",
"group" : "",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "BOOLEAN", "CHAR", "BYTE", "SHORT", "INTEGER", "LONG", "FLOAT", "DOUBLE", "DECIMAL", "DATE", "TIME", "DATETIME", "ZONED_DATETIME", "STRING", "BYTE_ARRAY" ],
"values" : [ "BOOLEAN", "CHAR", "BYTE", "SHORT", "INTEGER", "LONG", "FLOAT", "DOUBLE", "DECIMAL", "DATE", "TIME", "DATETIME", "ZONED_DATETIME", "STRING", "BYTE_ARRAY" ],
"configDefinitions" : null,
"modelType" : "VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.pipeline.config.PrimitiveFieldTypeChooserValues",
"filteringConfig" : ""
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 20,
"displayMode" : "BASIC",
"name" : "targetType",
"type" : "MODEL",
"defaultValue" : "INTEGER"
}, {
"fieldName" : "treatInputFieldAsDate",
"description" : "Select to convert input Long to DateTime before converting to a String",
"max" : 9223372036854775807,
"label" : "Treat Input Field as Date",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"targetType" : [ "STRING" ]
},
"dependsOn" : "",
"group" : "TYPE_CONVERSION",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 20,
"displayMode" : "BASIC",
"name" : "treatInputFieldAsDate",
"type" : "BOOLEAN",
"defaultValue" : false
}, {
"fieldName" : "dataLocale",
"description" : "Affects the interpretation of locale sensitive data, such as using the comma as a decimal separator",
"max" : 9223372036854775807,
"label" : "Data Locale",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"targetType" : [ "ZONED_DATETIME", "INTEGER", "LONG", "DOUBLE", "BYTE", "FLOAT", "SHORT", "DECIMAL" ]
},
"dependsOn" : "",
"group" : "TYPE_CONVERSION",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "Albanian", "Albanian (Albania)", "Arabic", "Arabic (Algeria)", "Arabic (Bahrain)", "Arabic (Egypt)", "Arabic (Iraq)", "Arabic (Jordan)", "Arabic (Kuwait)", "Arabic (Lebanon)", "Arabic (Libya)", "Arabic (Morocco)", "Arabic (Oman)", "Arabic (Qatar)", "Arabic (Saudi Arabia)", "Arabic (Sudan)", "Arabic (Syria)", "Arabic (Tunisia)", "Arabic (United Arab Emirates)", "Arabic (Yemen)", "Belarusian", "Belarusian (Belarus)", "Bulgarian", "Bulgarian (Bulgaria)", "Catalan", "Catalan (Spain)", "Chinese", "Chinese (China)", "Chinese (Hong Kong)", "Chinese (Singapore)", "Chinese (Taiwan)", "Croatian", "Croatian (Croatia)", "Czech", "Czech (Czech Republic)", "Danish", "Danish (Denmark)", "Dutch", "Dutch (Belgium)", "Dutch (Netherlands)", "English", "English (Australia)", "English (Canada)", "English (India)", "English (Ireland)", "English (Malta)", "English (New Zealand)", "English (Philippines)", "English (Singapore)", "English (South Africa)", "English (United Kingdom)", "English (United States)", "Estonian", "Estonian (Estonia)", "Finnish", "Finnish (Finland)", "French", "French (Belgium)", "French (Canada)", "French (France)", "French (Luxembourg)", "French (Switzerland)", "German", "German (Austria)", "German (Germany)", "German (Greece)", "German (Luxembourg)", "German (Switzerland)", "Greek", "Greek (Cyprus)", "Greek (Greece)", "Hebrew", "Hebrew (Israel)", "Hindi", "Hindi (India)", "Hungarian", "Hungarian (Hungary)", "Icelandic", "Icelandic (Iceland)", "Indonesian", "Indonesian (Indonesia)", "Irish", "Irish (Ireland)", "Italian", "Italian (Italy)", "Italian (Switzerland)", "Japanese", "Japanese (Japan)", "Japanese (Japan,JP)", "Korean", "Korean (South Korea)", "Latvian", "Latvian (Latvia)", "Lithuanian", "Lithuanian (Lithuania)", "Macedonian", "Macedonian (Macedonia)", "Malay", "Malay (Malaysia)", "Maltese", "Maltese (Malta)", "Norwegian", "Norwegian (Norway)", "Norwegian (Norway,Nynorsk)", "Polish", "Polish (Poland)", "Portuguese", "Portuguese (Brazil)", "Portuguese (Portugal)", "Romanian", "Romanian (Romania)", "Russian", "Russian (Russia)", "Serbian (Latin)", "Serbian (Latin,Bosnia and Herzegovina)", "Serbian (Montenegro)", "Serbian (Serbia and Montenegro)", "Serbian (Serbia)", "Slovak", "Slovak (Slovakia)", "Slovenian", "Slovenian (Slovenia)", "Spanish", "Spanish (Argentina)", "Spanish (Bolivia)", "Spanish (Chile)", "Spanish (Colombia)", "Spanish (Costa Rica)", "Spanish (Cuba)", "Spanish (Dominican Republic)", "Spanish (Ecuador)", "Spanish (El Salvador)", "Spanish (Guatemala)", "Spanish (Honduras)", "Spanish (Mexico)", "Spanish (Nicaragua)", "Spanish (Panama)", "Spanish (Paraguay)", "Spanish (Peru)", "Spanish (Puerto Rico)", "Spanish (Spain)", "Spanish (United States)", "Spanish (Uruguay)", "Spanish (Venezuela)", "Swedish", "Swedish (Sweden)", "Thai", "Thai (Thailand)", "Thai (Thailand,TH)", "Turkish", "Turkish (Turkey)", "Ukrainian", "Ukrainian (Ukraine)", "Vietnamese", "Vietnamese (Vietnam)" ],
"values" : [ "sq", "sq,AL", "ar", "ar,DZ", "ar,BH", "ar,EG", "ar,IQ", "ar,JO", "ar,KW", "ar,LB", "ar,LY", "ar,MA", "ar,OM", "ar,QA", "ar,SA", "ar,SD", "ar,SY", "ar,TN", "ar,AE", "ar,YE", "be", "be,BY", "bg", "bg,BG", "ca", "ca,ES", "zh", "zh,CN", "zh,HK", "zh,SG", "zh,TW", "hr", "hr,HR", "cs", "cs,CZ", "da", "da,DK", "nl", "nl,BE", "nl,NL", "en", "en,AU", "en,CA", "en,IN", "en,IE", "en,MT", "en,NZ", "en,PH", "en,SG", "en,ZA", "en,GB", "en,US", "et", "et,EE", "fi", "fi,FI", "fr", "fr,BE", "fr,CA", "fr,FR", "fr,LU", "fr,CH", "de", "de,AT", "de,DE", "de,GR", "de,LU", "de,CH", "el", "el,CY", "el,GR", "iw", "iw,IL", "hi", "hi,IN", "hu", "hu,HU", "is", "is,IS", "in", "in,ID", "ga", "ga,IE", "it", "it,IT", "it,CH", "ja", "ja,JP", "ja,JP,JP", "ko", "ko,KR", "lv", "lv,LV", "lt", "lt,LT", "mk", "mk,MK", "ms", "ms,MY", "mt", "mt,MT", "no", "no,NO", "no,NO,NY", "pl", "pl,PL", "pt", "pt,BR", "pt,PT", "ro", "ro,RO", "ru", "ru,RU", "sr", "sr,BA", "sr,ME", "sr,CS", "sr,RS", "sk", "sk,SK", "sl", "sl,SI", "es", "es,AR", "es,BO", "es,CL", "es,CO", "es,CR", "es,CU", "es,DO", "es,EC", "es,SV", "es,GT", "es,HN", "es,MX", "es,NI", "es,PA", "es,PY", "es,PE", "es,PR", "es,ES", "es,US", "es,UY", "es,VE", "sv", "sv,SE", "th", "th,TH", "th,TH,TH", "tr", "tr,TR", "uk", "uk,UA", "vi", "vi,VN" ],
"configDefinitions" : null,
"modelType" : "VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.pipeline.config.LocaleChooserValues",
"filteringConfig" : ""
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 30,
"displayMode" : "BASIC",
"name" : "dataLocale",
"type" : "MODEL",
"defaultValue" : "en,US"
}, {
"fieldName" : "scale",
"description" : "Decimal Value Scale",
"max" : 9223372036854775807,
"label" : "Scale",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : {
"targetType" : [ "DECIMAL" ]
},
"dependsOn" : "",
"group" : "TYPE_CONVERSION",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 40,
"displayMode" : "BASIC",
"name" : "scale",
"type" : "NUMBER",
"defaultValue" : -1
}, {
"fieldName" : "decimalScaleRoundingStrategy",
"description" : "Rounding strategy during scale conversion",
"max" : 9223372036854775807,
"label" : "Rounding Strategy",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : {
"targetType" : [ "DECIMAL" ]
},
"dependsOn" : "",
"group" : "TYPE_CONVERSION",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "Round up", "Round Down", "Round Ceil", "Round Floor", "Round Half Up", "Round Half Down", "Round Half Even", "Round Unnecessary" ],
"values" : [ "ROUND_UP", "ROUND_DOWN", "ROUND_CEILING", "ROUND_FLOOR", "ROUND_HALF_UP", "ROUND_HALF_DOWN", "ROUND_HALF_EVEN", "ROUND_UNNECESSARY" ],
"configDefinitions" : null,
"modelType" : "VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.pipeline.config.DecimalScaleRoundingStrategyChooserValues",
"filteringConfig" : ""
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 50,
"displayMode" : "BASIC",
"name" : "decimalScaleRoundingStrategy",
"type" : "MODEL",
"defaultValue" : "ROUND_UNNECESSARY"
}, {
"fieldName" : "dateFormat",
"description" : "Select or enter any valid date or datetime format",
"max" : 9223372036854775807,
"label" : "Date Format",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"targetType" : [ "DATETIME", "DATE", "STRING", "TIME" ]
},
"dependsOn" : "",
"group" : "TYPE_CONVERSION",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "yyyy-MM-dd", "dd-MMM-yyyy", "yyyy-MM-dd HH:mm:ss", "yyyy-MM-dd HH:mm:ss.SSS", "yyyy-MM-dd HH:mm:ss.SSS Z", "yyyy-MM-dd'T'HH:mm'Z'", "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", "Other ..." ],
"values" : [ "YYYY_MM_DD", "DD_MM_YYYY", "YYYY_MM_DD_HH_MM_SS", "YYYY_MM_DD_HH_MM_SS_SSS", "YYYY_MM_DD_HH_MM_SS_SSS_Z", "YYYY_MM_DD_T_HH_MM_Z", "YYYY_MM_DD_T_HH_MM_SS_SSS_Z", "OTHER" ],
"configDefinitions" : null,
"modelType" : "VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.pipeline.config.DateFormatChooserValues",
"filteringConfig" : ""
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 40,
"displayMode" : "BASIC",
"name" : "dateFormat",
"type" : "MODEL",
"defaultValue" : "YYYY_MM_DD"
}, {
"fieldName" : "otherDateFormat",
"description" : "",
"max" : 9223372036854775807,
"label" : "Other Date Format",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dateFormat" : [ "OTHER" ],
"targetType" : [ "DATETIME", "DATE", "STRING", "TIME" ]
},
"dependsOn" : "",
"group" : "TYPE_CONVERSION",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 50,
"displayMode" : "BASIC",
"name" : "otherDateFormat",
"type" : "STRING",
"defaultValue" : null
}, {
"fieldName" : "zonedDateTimeFormat",
"description" : "Select or enter any valid date or datetime format",
"max" : 9223372036854775807,
"label" : "Zoned DateTime Format",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"targetType" : [ "ZONED_DATETIME", "STRING" ]
},
"dependsOn" : "",
"group" : "TYPE_CONVERSION",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "yyyy-MM-dd'T'HH:mm:ssX", "yyyy-MM-dd'T'HH:mm:ssX[VV]", "Other ..." ],
"values" : [ "ISO_OFFSET_DATE_TIME", "ISO_ZONED_DATE_TIME", "OTHER" ],
"configDefinitions" : null,
"modelType" : "VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.pipeline.ZonedDateTimeFormatChooserValues",
"filteringConfig" : ""
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 60,
"displayMode" : "BASIC",
"name" : "zonedDateTimeFormat",
"type" : "MODEL",
"defaultValue" : "ISO_ZONED_DATE_TIME"
}, {
"fieldName" : "zonedDateTimeTargetTimeZone",
"description" : "Time Zone ",
"max" : 9223372036854775807,
"label" : "Target Time Zone",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"targetType" : [ "ZONED_DATETIME" ]
},
"dependsOn" : "",
"group" : "TYPE_CONVERSION",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "Same as Data Collector: +00:00 UTC (UTC)", "-11:00 ST (US/Samoa)", "-11:00 ST (Pacific/Samoa)", "-11:00 ST (Pacific/Pago_Pago)", "-11:00 ST (Pacific/Midway)", "-11:00 NUT (Pacific/Niue)", "-10:00 TAHT (Pacific/Tahiti)", "-10:00 HT (US/Hawaii)", "-10:00 HT (SystemV/HST10)", "-10:00 HT (Pacific/Johnston)", "-10:00 HT (Pacific/Honolulu)", "-10:00 CKT (Pacific/Rarotonga)", "-09:30 MART (Pacific/Marquesas)", "-09:00 HT (US/Aleutian)", "-09:00 HT (America/Atka)", "-09:00 HT (America/Adak)", "-09:00 GAMT (Pacific/Gambier)", "-09:00 AKT (SystemV/YST9YDT)", "-09:00 AKT (SystemV/YST9)", "-08:00 PT (SystemV/PST8PDT)", "-08:00 PT (SystemV/PST8)", "-08:00 PT (Pacific/Pitcairn)", "-08:00 AKT (US/Alaska)", "-08:00 AKT (America/Yakutat)", "-08:00 AKT (America/Sitka)", "-08:00 AKT (America/Nome)", "-08:00 AKT (America/Metlakatla)", "-08:00 AKT (America/Juneau)", "-08:00 AKT (America/Anchorage)", "-07:00 PT (US/Pacific-New)", "-07:00 PT (US/Pacific)", "-07:00 PT (PST8PDT)", "-07:00 PT (Mexico/BajaNorte)", "-07:00 PT (Canada/Yukon)", "-07:00 PT (Canada/Pacific)", "-07:00 PT (America/Whitehorse)", "-07:00 PT (America/Vancouver)", "-07:00 PT (America/Tijuana)", "-07:00 PT (America/Santa_Isabel)", "-07:00 PT (America/Los_Angeles)", "-07:00 PT (America/Ensenada)", "-07:00 PT (America/Dawson)", "-07:00 MT (US/Arizona)", "-07:00 MT (SystemV/MST7MDT)", "-07:00 MT (SystemV/MST7)", "-07:00 MT (America/Phoenix)", "-07:00 MT (America/Hermosillo)", "-07:00 MT (America/Fort_Nelson)", "-07:00 MT (America/Dawson_Creek)", "-07:00 MT (America/Creston)", "-06:00 MT (US/Mountain)", "-06:00 MT (Navajo)", "-06:00 MT (Mexico/BajaSur)", "-06:00 MT (MST7MDT)", "-06:00 MT (Canada/Mountain)", "-06:00 MT (America/Yellowknife)", "-06:00 MT (America/Shiprock)", "-06:00 MT (America/Ojinaga)", "-06:00 MT (America/Mazatlan)", "-06:00 MT (America/Inuvik)", "-06:00 MT (America/Edmonton)", "-06:00 MT (America/Denver)", "-06:00 MT (America/Chihuahua)", "-06:00 MT (America/Cambridge_Bay)", "-06:00 MT (America/Boise)", "-06:00 GALT (Pacific/Galapagos)", "-06:00 EAST (Pacific/Easter)", "-06:00 EAST (Chile/EasterIsland)", "-06:00 CT (SystemV/CST6CDT)", "-06:00 CT (SystemV/CST6)", "-06:00 CT (Canada/Saskatchewan)", "-06:00 CT (America/Tegucigalpa)", "-06:00 CT (America/Swift_Current)", "-06:00 CT (America/Regina)", "-06:00 CT (America/Managua)", "-06:00 CT (America/Guatemala)", "-06:00 CT (America/El_Salvador)", "-06:00 CT (America/Costa_Rica)", "-06:00 CT (America/Belize)", "-05:00 PET (America/Lima)", "-05:00 ET (SystemV/EST5EDT)", "-05:00 ET (SystemV/EST5)", "-05:00 ET (Jamaica)", "-05:00 ET (America/Panama)", "-05:00 ET (America/Jamaica)", "-05:00 ET (America/Coral_Harbour)", "-05:00 ET (America/Cayman)", "-05:00 ET (America/Cancun)", "-05:00 ET (America/Atikokan)", "-05:00 ECT (America/Guayaquil)", "-05:00 CT (US/Indiana-Starke)", "-05:00 CT (US/Central)", "-05:00 CT (Mexico/General)", "-05:00 CT (Canada/Central)", "-05:00 CT (CST6CDT)", "-05:00 CT (America/Winnipeg)", "-05:00 CT (America/Resolute)", "-05:00 CT (America/Rankin_Inlet)", "-05:00 CT (America/Rainy_River)", "-05:00 CT (America/North_Dakota/New_Salem)", "-05:00 CT (America/North_Dakota/Center)", "-05:00 CT (America/North_Dakota/Beulah)", "-05:00 CT (America/Monterrey)", "-05:00 CT (America/Mexico_City)", "-05:00 CT (America/Merida)", "-05:00 CT (America/Menominee)", "-05:00 CT (America/Matamoros)", "-05:00 CT (America/Knox_IN)", "-05:00 CT (America/Indiana/Tell_City)", "-05:00 CT (America/Indiana/Knox)", "-05:00 CT (America/Chicago)", "-05:00 CT (America/Bahia_Banderas)", "-05:00 COT (America/Bogota)", "-05:00 ACT (Brazil/Acre)", "-05:00 ACT (America/Rio_Branco)", "-05:00 ACT (America/Porto_Acre)", "-05:00 ACT (America/Eirunepe)", "-04:00 VET (America/Caracas)", "-04:00 PYT (America/Asuncion)", "-04:00 GYT (America/Guyana)", "-04:00 ET (US/Michigan)", "-04:00 ET (US/Eastern)", "-04:00 ET (US/East-Indiana)", "-04:00 ET (EST5EDT)", "-04:00 ET (Canada/Eastern)", "-04:00 ET (America/Toronto)", "-04:00 ET (America/Thunder_Bay)", "-04:00 ET (America/Port-au-Prince)", "-04:00 ET (America/Pangnirtung)", "-04:00 ET (America/Nipigon)", "-04:00 ET (America/New_York)", "-04:00 ET (America/Nassau)", "-04:00 ET (America/Montreal)", "-04:00 ET (America/Louisville)", "-04:00 ET (America/Kentucky/Monticello)", "-04:00 ET (America/Kentucky/Louisville)", "-04:00 ET (America/Iqaluit)", "-04:00 ET (America/Indianapolis)", "-04:00 ET (America/Indiana/Winamac)", "-04:00 ET (America/Indiana/Vincennes)", "-04:00 ET (America/Indiana/Vevay)", "-04:00 ET (America/Indiana/Petersburg)", "-04:00 ET (America/Indiana/Marengo)", "-04:00 ET (America/Indiana/Indianapolis)", "-04:00 ET (America/Grand_Turk)", "-04:00 ET (America/Fort_Wayne)", "-04:00 ET (America/Detroit)", "-04:00 CT (Cuba)", "-04:00 CT (America/Havana)", "-04:00 CLT (Chile/Continental)", "-04:00 CLT (America/Santiago)", "-04:00 BOT (America/La_Paz)", "-04:00 AT (SystemV/AST4ADT)", "-04:00 AT (SystemV/AST4)", "-04:00 AT (America/Virgin)", "-04:00 AT (America/Tortola)", "-04:00 AT (America/St_Vincent)", "-04:00 AT (America/St_Thomas)", "-04:00 AT (America/St_Lucia)", "-04:00 AT (America/St_Kitts)", "-04:00 AT (America/St_Barthelemy)", "-04:00 AT (America/Santo_Domingo)", "-04:00 AT (America/Puerto_Rico)", "-04:00 AT (America/Port_of_Spain)", "-04:00 AT (America/Montserrat)", "-04:00 AT (America/Martinique)", "-04:00 AT (America/Marigot)", "-04:00 AT (America/Lower_Princes)", "-04:00 AT (America/Kralendijk)", "-04:00 AT (America/Guadeloupe)", "-04:00 AT (America/Grenada)", "-04:00 AT (America/Dominica)", "-04:00 AT (America/Curacao)", "-04:00 AT (America/Blanc-Sablon)", "-04:00 AT (America/Barbados)", "-04:00 AT (America/Aruba)", "-04:00 AT (America/Antigua)", "-04:00 AT (America/Anguilla)", "-04:00 AMT (Brazil/West)", "-04:00 AMT (America/Porto_Velho)", "-04:00 AMT (America/Manaus)", "-04:00 AMT (America/Cuiaba)", "-04:00 AMT (America/Campo_Grande)", "-04:00 AMT (America/Boa_Vista)", "-03:00 UYT (America/Montevideo)", "-03:00 SRT (America/Paramaribo)", "-03:00 ROTT (Antarctica/Rothera)", "-03:00 GFT (America/Cayenne)", "-03:00 FKT (Atlantic/Stanley)", "-03:00 CLT (Antarctica/Palmer)", "-03:00 BRT (Brazil/East)", "-03:00 BRT (America/Sao_Paulo)", "-03:00 BRT (America/Santarem)", "-03:00 BRT (America/Recife)", "-03:00 BRT (America/Maceio)", "-03:00 BRT (America/Fortaleza)", "-03:00 BRT (America/Belem)", "-03:00 BRT (America/Bahia)", "-03:00 BRT (America/Araguaina)", "-03:00 America/Punta_Arenas (America/Punta_Arenas)", "-03:00 AT (Canada/Atlantic)", "-03:00 AT (Atlantic/Bermuda)", "-03:00 AT (America/Thule)", "-03:00 AT (America/Moncton)", "-03:00 AT (America/Halifax)", "-03:00 AT (America/Goose_Bay)", "-03:00 AT (America/Glace_Bay)", "-03:00 ART (America/Rosario)", "-03:00 ART (America/Mendoza)", "-03:00 ART (America/Jujuy)", "-03:00 ART (America/Cordoba)", "-03:00 ART (America/Catamarca)", "-03:00 ART (America/Buenos_Aires)", "-03:00 ART (America/Argentina/Ushuaia)", "-03:00 ART (America/Argentina/Tucuman)", "-03:00 ART (America/Argentina/San_Luis)", "-03:00 ART (America/Argentina/San_Juan)", "-03:00 ART (America/Argentina/Salta)", "-03:00 ART (America/Argentina/Rio_Gallegos)", "-03:00 ART (America/Argentina/Mendoza)", "-03:00 ART (America/Argentina/La_Rioja)", "-03:00 ART (America/Argentina/Jujuy)", "-03:00 ART (America/Argentina/Cordoba)", "-03:00 ART (America/Argentina/ComodRivadavia)", "-03:00 ART (America/Argentina/Catamarca)", "-03:00 ART (America/Argentina/Buenos_Aires)", "-02:30 NT (Canada/Newfoundland)", "-02:30 NT (America/St_Johns)", "-02:00 WGT (America/Godthab)", "-02:00 PMT (America/Miquelon)", "-02:00 GT (Atlantic/South_Georgia)", "-02:00 FNT (Brazil/DeNoronha)", "-02:00 FNT (America/Noronha)", "-01:00 CVT (Atlantic/Cape_Verde)", "+00:00 AZOT (Atlantic/Azores)", "+00:00 EGT (America/Scoresbysund)", "+00:00 GMT (Africa/Abidjan)", "+00:00 GMT (Africa/Accra)", "+00:00 GMT (Africa/Bamako)", "+00:00 GMT (Africa/Banjul)", "+00:00 GMT (Africa/Bissau)", "+00:00 GMT (Africa/Conakry)", "+00:00 GMT (Africa/Dakar)", "+00:00 GMT (Africa/Freetown)", "+00:00 GMT (Africa/Lome)", "+00:00 GMT (Africa/Monrovia)", "+00:00 GMT (Africa/Nouakchott)", "+00:00 GMT (Africa/Ouagadougou)", "+00:00 GMT (Africa/Sao_Tome)", "+00:00 GMT (Africa/Timbuktu)", "+00:00 GMT (America/Danmarkshavn)", "+00:00 GMT (Atlantic/Reykjavik)", "+00:00 GMT (Atlantic/St_Helena)", "+00:00 GMT (Etc/Greenwich)", "+00:00 GMT (GMT)", "+00:00 GMT (GMT0)", "+00:00 GMT (Greenwich)", "+00:00 GMT (Iceland)", "+00:00 UTC (Etc/UCT)", "+00:00 UTC (Etc/UTC)", "+00:00 UTC (Etc/Universal)", "+00:00 UTC (Etc/Zulu)", "+00:00 UTC (UCT)", "+00:00 UTC (Universal)", "+00:00 UTC (Zulu)", "+01:00 BT (Europe/Belfast)", "+01:00 BT (Europe/Guernsey)", "+01:00 BT (Europe/Isle_of_Man)", "+01:00 BT (Europe/Jersey)", "+01:00 BT (Europe/London)", "+01:00 BT (GB)", "+01:00 BT (GB-Eire)", "+01:00 CET (Africa/Algiers)", "+01:00 CET (Africa/Tunis)", "+01:00 IT (Eire)", "+01:00 IT (Europe/Dublin)", "+01:00 WAT (Africa/Bangui)", "+01:00 WAT (Africa/Brazzaville)", "+01:00 WAT (Africa/Douala)", "+01:00 WAT (Africa/Kinshasa)", "+01:00 WAT (Africa/Lagos)", "+01:00 WAT (Africa/Libreville)", "+01:00 WAT (Africa/Luanda)", "+01:00 WAT (Africa/Malabo)", "+01:00 WAT (Africa/Ndjamena)", "+01:00 WAT (Africa/Niamey)", "+01:00 WAT (Africa/Porto-Novo)", "+01:00 WET (Africa/Casablanca)", "+01:00 WET (Africa/El_Aaiun)", "+01:00 WET (Atlantic/Canary)", "+01:00 WET (Atlantic/Faeroe)", "+01:00 WET (Atlantic/Faroe)", "+01:00 WET (Atlantic/Madeira)", "+01:00 WET (Europe/Lisbon)", "+01:00 WET (Portugal)", "+01:00 WET (WET)", "+02:00 ATT (Antarctica/Troll)", "+02:00 CAT (Africa/Blantyre)", "+02:00 CAT (Africa/Bujumbura)", "+02:00 CAT (Africa/Gaborone)", "+02:00 CAT (Africa/Harare)", "+02:00 CAT (Africa/Khartoum)", "+02:00 CAT (Africa/Kigali)", "+02:00 CAT (Africa/Lubumbashi)", "+02:00 CAT (Africa/Lusaka)", "+02:00 CAT (Africa/Maputo)", "+02:00 CAT (Africa/Windhoek)", "+02:00 CET (Africa/Ceuta)", "+02:00 CET (Arctic/Longyearbyen)", "+02:00 CET (Atlantic/Jan_Mayen)", "+02:00 CET (CET)", "+02:00 CET (Europe/Amsterdam)", "+02:00 CET (Europe/Andorra)", "+02:00 CET (Europe/Belgrade)", "+02:00 CET (Europe/Berlin)", "+02:00 CET (Europe/Bratislava)", "+02:00 CET (Europe/Brussels)", "+02:00 CET (Europe/Budapest)", "+02:00 CET (Europe/Busingen)", "+02:00 CET (Europe/Copenhagen)", "+02:00 CET (Europe/Gibraltar)", "+02:00 CET (Europe/Ljubljana)", "+02:00 CET (Europe/Luxembourg)", "+02:00 CET (Europe/Madrid)", "+02:00 CET (Europe/Malta)", "+02:00 CET (Europe/Monaco)", "+02:00 CET (Europe/Oslo)", "+02:00 CET (Europe/Paris)", "+02:00 CET (Europe/Podgorica)", "+02:00 CET (Europe/Prague)", "+02:00 CET (Europe/Rome)", "+02:00 CET (Europe/San_Marino)", "+02:00 CET (Europe/Sarajevo)", "+02:00 CET (Europe/Skopje)", "+02:00 CET (Europe/Stockholm)", "+02:00 CET (Europe/Tirane)", "+02:00 CET (Europe/Vaduz)", "+02:00 CET (Europe/Vatican)", "+02:00 CET (Europe/Vienna)", "+02:00 CET (Europe/Warsaw)", "+02:00 CET (Europe/Zagreb)", "+02:00 CET (Europe/Zurich)", "+02:00 CET (Poland)", "+02:00 EET (Africa/Cairo)", "+02:00 EET (Africa/Tripoli)", "+02:00 EET (Egypt)", "+02:00 EET (Europe/Kaliningrad)", "+02:00 EET (Libya)", "+02:00 MET (MET)", "+02:00 SAT (Africa/Johannesburg)", "+02:00 SAT (Africa/Maseru)", "+02:00 SAT (Africa/Mbabane)", "+03:00 AT (Asia/Aden)", "+03:00 AT (Asia/Baghdad)", "+03:00 AT (Asia/Bahrain)", "+03:00 AT (Asia/Kuwait)", "+03:00 AT (Asia/Qatar)", "+03:00 AT (Asia/Riyadh)", "+03:00 Asia/Famagusta (Asia/Famagusta)", "+03:00 EAT (Africa/Addis_Ababa)", "+03:00 EAT (Africa/Asmara)", "+03:00 EAT (Africa/Asmera)", "+03:00 EAT (Africa/Dar_es_Salaam)", "+03:00 EAT (Africa/Djibouti)", "+03:00 EAT (Africa/Juba)", "+03:00 EAT (Africa/Kampala)", "+03:00 EAT (Africa/Mogadishu)", "+03:00 EAT (Africa/Nairobi)", "+03:00 EAT (Indian/Antananarivo)", "+03:00 EAT (Indian/Comoro)", "+03:00 EAT (Indian/Mayotte)", "+03:00 EET (Asia/Amman)", "+03:00 EET (Asia/Beirut)", "+03:00 EET (Asia/Damascus)", "+03:00 EET (Asia/Gaza)", "+03:00 EET (Asia/Hebron)", "+03:00 EET (Asia/Istanbul)", "+03:00 EET (Asia/Nicosia)", "+03:00 EET (EET)", "+03:00 EET (Europe/Athens)", "+03:00 EET (Europe/Bucharest)", "+03:00 EET (Europe/Chisinau)", "+03:00 EET (Europe/Helsinki)", "+03:00 EET (Europe/Istanbul)", "+03:00 EET (Europe/Kiev)", "+03:00 EET (Europe/Mariehamn)", "+03:00 EET (Europe/Nicosia)", "+03:00 EET (Europe/Riga)", "+03:00 EET (Europe/Sofia)", "+03:00 EET (Europe/Tallinn)", "+03:00 EET (Europe/Tiraspol)", "+03:00 EET (Europe/Uzhgorod)", "+03:00 EET (Europe/Vilnius)", "+03:00 EET (Europe/Zaporozhye)", "+03:00 EET (Turkey)", "+03:00 Europe/Kirov (Europe/Kirov)", "+03:00 IT (Asia/Jerusalem)", "+03:00 IT (Asia/Tel_Aviv)", "+03:00 IT (Israel)", "+03:00 MT (Europe/Minsk)", "+03:00 MT (Europe/Moscow)", "+03:00 MT (Europe/Simferopol)", "+03:00 MT (W-SU)", "+03:00 SYOT (Antarctica/Syowa)", "+04:00 AMT (Asia/Yerevan)", "+04:00 AZT (Asia/Baku)", "+04:00 Europe/Astrakhan (Europe/Astrakhan)", "+04:00 Europe/Saratov (Europe/Saratov)", "+04:00 Europe/Ulyanovsk (Europe/Ulyanovsk)", "+04:00 GET (Asia/Tbilisi)", "+04:00 GT (Asia/Dubai)", "+04:00 GT (Asia/Muscat)", "+04:00 MT (Europe/Volgograd)", "+04:00 MUT (Indian/Mauritius)", "+04:00 RET (Indian/Reunion)", "+04:00 SAMT (Europe/Samara)", "+04:00 SCT (Indian/Mahe)", "+04:30 AFT (Asia/Kabul)", "+04:30 IRT (Asia/Tehran)", "+04:30 IRT (Iran)", "+05:00 AQTT (Asia/Aqtau)", "+05:00 AQTT (Asia/Aqtobe)", "+05:00 Asia/Atyrau (Asia/Atyrau)", "+05:00 MAWT (Antarctica/Mawson)", "+05:00 MVT (Indian/Maldives)", "+05:00 ORAT (Asia/Oral)", "+05:00 PKT (Asia/Karachi)", "+05:00 QYZT (Asia/Qyzylorda)", "+05:00 TFT (Indian/Kerguelen)", "+05:00 TJT (Asia/Dushanbe)", "+05:00 TMT (Asia/Ashgabat)", "+05:00 TMT (Asia/Ashkhabad)", "+05:00 UZT (Asia/Samarkand)", "+05:00 UZT (Asia/Tashkent)", "+05:00 YEKT (Asia/Yekaterinburg)", "+05:30 IT (Asia/Calcutta)", "+05:30 IT (Asia/Colombo)", "+05:30 IT (Asia/Kolkata)", "+05:45 NPT (Asia/Kathmandu)", "+05:45 NPT (Asia/Katmandu)", "+06:00 ALMT (Asia/Almaty)", "+06:00 BDT (Asia/Dacca)", "+06:00 BDT (Asia/Dhaka)", "+06:00 BTT (Asia/Thimbu)", "+06:00 BTT (Asia/Thimphu)", "+06:00 IOT (Indian/Chagos)", "+06:00 KGT (Asia/Bishkek)", "+06:00 OMST (Asia/Omsk)", "+06:00 QOST (Asia/Qostanay)", "+06:00 VOST (Antarctica/Vostok)", "+06:00 XJT (Asia/Kashgar)", "+06:00 XJT (Asia/Urumqi)", "+06:30 CCT (Indian/Cocos)", "+06:30 MMT (Asia/Rangoon)", "+06:30 MMT (Asia/Yangon)", "+07:00 Asia/Barnaul (Asia/Barnaul)", "+07:00 Asia/Tomsk (Asia/Tomsk)", "+07:00 CIT (Indian/Christmas)", "+07:00 DAVT (Antarctica/Davis)", "+07:00 HOVT (Asia/Hovd)", "+07:00 ICT (Asia/Bangkok)", "+07:00 ICT (Asia/Ho_Chi_Minh)", "+07:00 ICT (Asia/Phnom_Penh)", "+07:00 ICT (Asia/Saigon)", "+07:00 ICT (Asia/Vientiane)", "+07:00 KRAT (Asia/Krasnoyarsk)", "+07:00 KRAT (Asia/Novokuznetsk)", "+07:00 NOVT (Asia/Novosibirsk)", "+07:00 WIB (Asia/Jakarta)", "+07:00 WIB (Asia/Pontianak)", "+08:00 AWT (Antarctica/Casey)", "+08:00 AWT (Australia/Perth)", "+08:00 AWT (Australia/West)", "+08:00 BNT (Asia/Brunei)", "+08:00 CHOT (Asia/Choibalsan)", "+08:00 CT (Asia/Chongqing)", "+08:00 CT (Asia/Chungking)", "+08:00 CT (Asia/Harbin)", "+08:00 CT (Asia/Macao)", "+08:00 CT (Asia/Macau)", "+08:00 CT (Asia/Shanghai)", "+08:00 CT (Asia/Taipei)", "+08:00 CT (PRC)", "+08:00 HKT (Asia/Hong_Kong)", "+08:00 HKT (Hongkong)", "+08:00 IRKT (Asia/Irkutsk)", "+08:00 MYT (Asia/Kuala_Lumpur)", "+08:00 MYT (Asia/Kuching)", "+08:00 PT (Asia/Manila)", "+08:00 SGT (Asia/Singapore)", "+08:00 SGT (Singapore)", "+08:00 ULAT (Asia/Ulaanbaatar)", "+08:00 ULAT (Asia/Ulan_Bator)", "+08:00 WITA (Asia/Makassar)", "+08:00 WITA (Asia/Ujung_Pandang)", "+08:45 ACWT (Australia/Eucla)", "+09:00 JT (Asia/Tokyo)", "+09:00 JT (Japan)", "+09:00 KT (Asia/Pyongyang)", "+09:00 KT (Asia/Seoul)", "+09:00 KT (ROK)", "+09:00 PWT (Pacific/Palau)", "+09:00 TLT (Asia/Dili)", "+09:00 WIT (Asia/Jayapura)", "+09:00 YAKT (Asia/Chita)", "+09:00 YAKT (Asia/Khandyga)", "+09:00 YAKT (Asia/Yakutsk)", "+09:30 ACT (Australia/Adelaide)", "+09:30 ACT (Australia/Broken_Hill)", "+09:30 ACT (Australia/Darwin)", "+09:30 ACT (Australia/North)", "+09:30 ACT (Australia/South)", "+09:30 ACT (Australia/Yancowinna)", "+10:00 AET (Australia/ACT)", "+10:00 AET (Australia/Brisbane)", "+10:00 AET (Australia/Canberra)", "+10:00 AET (Australia/Currie)", "+10:00 AET (Australia/Hobart)", "+10:00 AET (Australia/Lindeman)", "+10:00 AET (Australia/Melbourne)", "+10:00 AET (Australia/NSW)", "+10:00 AET (Australia/Queensland)", "+10:00 AET (Australia/Sydney)", "+10:00 AET (Australia/Tasmania)", "+10:00 AET (Australia/Victoria)", "+10:00 CHUT (Pacific/Chuuk)", "+10:00 CHUT (Pacific/Truk)", "+10:00 CHUT (Pacific/Yap)", "+10:00 ChT (Pacific/Guam)", "+10:00 ChT (Pacific/Saipan)", "+10:00 DDUT (Antarctica/DumontDUrville)", "+10:00 PGT (Pacific/Port_Moresby)", "+10:00 VLAT (Asia/Ust-Nera)", "+10:00 VLAT (Asia/Vladivostok)", "+10:30 LHT (Australia/LHI)", "+10:30 LHT (Australia/Lord_Howe)", "+11:00 BT (Pacific/Bougainville)", "+11:00 KOST (Pacific/Kosrae)", "+11:00 MAGT (Asia/Magadan)", "+11:00 MIST (Antarctica/Macquarie)", "+11:00 NCT (Pacific/Noumea)", "+11:00 NFT (Pacific/Norfolk)", "+11:00 PONT (Pacific/Pohnpei)", "+11:00 PONT (Pacific/Ponape)", "+11:00 SAKT (Asia/Sakhalin)", "+11:00 SBT (Pacific/Guadalcanal)", "+11:00 SRET (Asia/Srednekolymsk)", "+11:00 VUT (Pacific/Efate)", "+12:00 ANAT (Asia/Anadyr)", "+12:00 FJT (Pacific/Fiji)", "+12:00 GILT (Pacific/Tarawa)", "+12:00 MHT (Kwajalein)", "+12:00 MHT (Pacific/Kwajalein)", "+12:00 MHT (Pacific/Majuro)", "+12:00 NRT (Pacific/Nauru)", "+12:00 NZT (Antarctica/McMurdo)", "+12:00 NZT (Antarctica/South_Pole)", "+12:00 NZT (NZ)", "+12:00 NZT (Pacific/Auckland)", "+12:00 PETT (Asia/Kamchatka)", "+12:00 TVT (Pacific/Funafuti)", "+12:00 WAKT (Pacific/Wake)", "+12:00 WFT (Pacific/Wallis)", "+12:45 CHAT (NZ-CHAT)", "+12:45 CHAT (Pacific/Chatham)", "+13:00 PHOT (Pacific/Enderbury)", "+13:00 TKT (Pacific/Fakaofo)", "+13:00 TOT (Pacific/Tongatapu)", "+13:00 WST (Pacific/Apia)", "+14:00 LINT (Pacific/Kiritimati)" ],
"values" : [ "UTC", "US/Samoa", "Pacific/Samoa", "Pacific/Pago_Pago", "Pacific/Midway", "Pacific/Niue", "Pacific/Tahiti", "US/Hawaii", "SystemV/HST10", "Pacific/Johnston", "Pacific/Honolulu", "Pacific/Rarotonga", "Pacific/Marquesas", "US/Aleutian", "America/Atka", "America/Adak", "Pacific/Gambier", "SystemV/YST9YDT", "SystemV/YST9", "SystemV/PST8PDT", "SystemV/PST8", "Pacific/Pitcairn", "US/Alaska", "America/Yakutat", "America/Sitka", "America/Nome", "America/Metlakatla", "America/Juneau", "America/Anchorage", "US/Pacific-New", "US/Pacific", "PST8PDT", "Mexico/BajaNorte", "Canada/Yukon", "Canada/Pacific", "America/Whitehorse", "America/Vancouver", "America/Tijuana", "America/Santa_Isabel", "America/Los_Angeles", "America/Ensenada", "America/Dawson", "US/Arizona", "SystemV/MST7MDT", "SystemV/MST7", "America/Phoenix", "America/Hermosillo", "America/Fort_Nelson", "America/Dawson_Creek", "America/Creston", "US/Mountain", "Navajo", "Mexico/BajaSur", "MST7MDT", "Canada/Mountain", "America/Yellowknife", "America/Shiprock", "America/Ojinaga", "America/Mazatlan", "America/Inuvik", "America/Edmonton", "America/Denver", "America/Chihuahua", "America/Cambridge_Bay", "America/Boise", "Pacific/Galapagos", "Pacific/Easter", "Chile/EasterIsland", "SystemV/CST6CDT", "SystemV/CST6", "Canada/Saskatchewan", "America/Tegucigalpa", "America/Swift_Current", "America/Regina", "America/Managua", "America/Guatemala", "America/El_Salvador", "America/Costa_Rica", "America/Belize", "America/Lima", "SystemV/EST5EDT", "SystemV/EST5", "Jamaica", "America/Panama", "America/Jamaica", "America/Coral_Harbour", "America/Cayman", "America/Cancun", "America/Atikokan", "America/Guayaquil", "US/Indiana-Starke", "US/Central", "Mexico/General", "Canada/Central", "CST6CDT", "America/Winnipeg", "America/Resolute", "America/Rankin_Inlet", "America/Rainy_River", "America/North_Dakota/New_Salem", "America/North_Dakota/Center", "America/North_Dakota/Beulah", "America/Monterrey", "America/Mexico_City", "America/Merida", "America/Menominee", "America/Matamoros", "America/Knox_IN", "America/Indiana/Tell_City", "America/Indiana/Knox", "America/Chicago", "America/Bahia_Banderas", "America/Bogota", "Brazil/Acre", "America/Rio_Branco", "America/Porto_Acre", "America/Eirunepe", "America/Caracas", "America/Asuncion", "America/Guyana", "US/Michigan", "US/Eastern", "US/East-Indiana", "EST5EDT", "Canada/Eastern", "America/Toronto", "America/Thunder_Bay", "America/Port-au-Prince", "America/Pangnirtung", "America/Nipigon", "America/New_York", "America/Nassau", "America/Montreal", "America/Louisville", "America/Kentucky/Monticello", "America/Kentucky/Louisville", "America/Iqaluit", "America/Indianapolis", "America/Indiana/Winamac", "America/Indiana/Vincennes", "America/Indiana/Vevay", "America/Indiana/Petersburg", "America/Indiana/Marengo", "America/Indiana/Indianapolis", "America/Grand_Turk", "America/Fort_Wayne", "America/Detroit", "Cuba", "America/Havana", "Chile/Continental", "America/Santiago", "America/La_Paz", "SystemV/AST4ADT", "SystemV/AST4", "America/Virgin", "America/Tortola", "America/St_Vincent", "America/St_Thomas", "America/St_Lucia", "America/St_Kitts", "America/St_Barthelemy", "America/Santo_Domingo", "America/Puerto_Rico", "America/Port_of_Spain", "America/Montserrat", "America/Martinique", "America/Marigot", "America/Lower_Princes", "America/Kralendijk", "America/Guadeloupe", "America/Grenada", "America/Dominica", "America/Curacao", "America/Blanc-Sablon", "America/Barbados", "America/Aruba", "America/Antigua", "America/Anguilla", "Brazil/West", "America/Porto_Velho", "America/Manaus", "America/Cuiaba", "America/Campo_Grande", "America/Boa_Vista", "America/Montevideo", "America/Paramaribo", "Antarctica/Rothera", "America/Cayenne", "Atlantic/Stanley", "Antarctica/Palmer", "Brazil/East", "America/Sao_Paulo", "America/Santarem", "America/Recife", "America/Maceio", "America/Fortaleza", "America/Belem", "America/Bahia", "America/Araguaina", "America/Punta_Arenas", "Canada/Atlantic", "Atlantic/Bermuda", "America/Thule", "America/Moncton", "America/Halifax", "America/Goose_Bay", "America/Glace_Bay", "America/Rosario", "America/Mendoza", "America/Jujuy", "America/Cordoba", "America/Catamarca", "America/Buenos_Aires", "America/Argentina/Ushuaia", "America/Argentina/Tucuman", "America/Argentina/San_Luis", "America/Argentina/San_Juan", "America/Argentina/Salta", "America/Argentina/Rio_Gallegos", "America/Argentina/Mendoza", "America/Argentina/La_Rioja", "America/Argentina/Jujuy", "America/Argentina/Cordoba", "America/Argentina/ComodRivadavia", "America/Argentina/Catamarca", "America/Argentina/Buenos_Aires", "Canada/Newfoundland", "America/St_Johns", "America/Godthab", "America/Miquelon", "Atlantic/South_Georgia", "Brazil/DeNoronha", "America/Noronha", "Atlantic/Cape_Verde", "Atlantic/Azores", "America/Scoresbysund", "Africa/Abidjan", "Africa/Accra", "Africa/Bamako", "Africa/Banjul", "Africa/Bissau", "Africa/Conakry", "Africa/Dakar", "Africa/Freetown", "Africa/Lome", "Africa/Monrovia", "Africa/Nouakchott", "Africa/Ouagadougou", "Africa/Sao_Tome", "Africa/Timbuktu", "America/Danmarkshavn", "Atlantic/Reykjavik", "Atlantic/St_Helena", "Etc/Greenwich", "GMT", "GMT0", "Greenwich", "Iceland", "Etc/UCT", "Etc/UTC", "Etc/Universal", "Etc/Zulu", "UCT", "Universal", "Zulu", "Europe/Belfast", "Europe/Guernsey", "Europe/Isle_of_Man", "Europe/Jersey", "Europe/London", "GB", "GB-Eire", "Africa/Algiers", "Africa/Tunis", "Eire", "Europe/Dublin", "Africa/Bangui", "Africa/Brazzaville", "Africa/Douala", "Africa/Kinshasa", "Africa/Lagos", "Africa/Libreville", "Africa/Luanda", "Africa/Malabo", "Africa/Ndjamena", "Africa/Niamey", "Africa/Porto-Novo", "Africa/Casablanca", "Africa/El_Aaiun", "Atlantic/Canary", "Atlantic/Faeroe", "Atlantic/Faroe", "Atlantic/Madeira", "Europe/Lisbon", "Portugal", "WET", "Antarctica/Troll", "Africa/Blantyre", "Africa/Bujumbura", "Africa/Gaborone", "Africa/Harare", "Africa/Khartoum", "Africa/Kigali", "Africa/Lubumbashi", "Africa/Lusaka", "Africa/Maputo", "Africa/Windhoek", "Africa/Ceuta", "Arctic/Longyearbyen", "Atlantic/Jan_Mayen", "CET", "Europe/Amsterdam", "Europe/Andorra", "Europe/Belgrade", "Europe/Berlin", "Europe/Bratislava", "Europe/Brussels", "Europe/Budapest", "Europe/Busingen", "Europe/Copenhagen", "Europe/Gibraltar", "Europe/Ljubljana", "Europe/Luxembourg", "Europe/Madrid", "Europe/Malta", "Europe/Monaco", "Europe/Oslo", "Europe/Paris", "Europe/Podgorica", "Europe/Prague", "Europe/Rome", "Europe/San_Marino", "Europe/Sarajevo", "Europe/Skopje", "Europe/Stockholm", "Europe/Tirane", "Europe/Vaduz", "Europe/Vatican", "Europe/Vienna", "Europe/Warsaw", "Europe/Zagreb", "Europe/Zurich", "Poland", "Africa/Cairo", "Africa/Tripoli", "Egypt", "Europe/Kaliningrad", "Libya", "MET", "Africa/Johannesburg", "Africa/Maseru", "Africa/Mbabane", "Asia/Aden", "Asia/Baghdad", "Asia/Bahrain", "Asia/Kuwait", "Asia/Qatar", "Asia/Riyadh", "Asia/Famagusta", "Africa/Addis_Ababa", "Africa/Asmara", "Africa/Asmera", "Africa/Dar_es_Salaam", "Africa/Djibouti", "Africa/Juba", "Africa/Kampala", "Africa/Mogadishu", "Africa/Nairobi", "Indian/Antananarivo", "Indian/Comoro", "Indian/Mayotte", "Asia/Amman", "Asia/Beirut", "Asia/Damascus", "Asia/Gaza", "Asia/Hebron", "Asia/Istanbul", "Asia/Nicosia", "EET", "Europe/Athens", "Europe/Bucharest", "Europe/Chisinau", "Europe/Helsinki", "Europe/Istanbul", "Europe/Kiev", "Europe/Mariehamn", "Europe/Nicosia", "Europe/Riga", "Europe/Sofia", "Europe/Tallinn", "Europe/Tiraspol", "Europe/Uzhgorod", "Europe/Vilnius", "Europe/Zaporozhye", "Turkey", "Europe/Kirov", "Asia/Jerusalem", "Asia/Tel_Aviv", "Israel", "Europe/Minsk", "Europe/Moscow", "Europe/Simferopol", "W-SU", "Antarctica/Syowa", "Asia/Yerevan", "Asia/Baku", "Europe/Astrakhan", "Europe/Saratov", "Europe/Ulyanovsk", "Asia/Tbilisi", "Asia/Dubai", "Asia/Muscat", "Europe/Volgograd", "Indian/Mauritius", "Indian/Reunion", "Europe/Samara", "Indian/Mahe", "Asia/Kabul", "Asia/Tehran", "Iran", "Asia/Aqtau", "Asia/Aqtobe", "Asia/Atyrau", "Antarctica/Mawson", "Indian/Maldives", "Asia/Oral", "Asia/Karachi", "Asia/Qyzylorda", "Indian/Kerguelen", "Asia/Dushanbe", "Asia/Ashgabat", "Asia/Ashkhabad", "Asia/Samarkand", "Asia/Tashkent", "Asia/Yekaterinburg", "Asia/Calcutta", "Asia/Colombo", "Asia/Kolkata", "Asia/Kathmandu", "Asia/Katmandu", "Asia/Almaty", "Asia/Dacca", "Asia/Dhaka", "Asia/Thimbu", "Asia/Thimphu", "Indian/Chagos", "Asia/Bishkek", "Asia/Omsk", "Asia/Qostanay", "Antarctica/Vostok", "Asia/Kashgar", "Asia/Urumqi", "Indian/Cocos", "Asia/Rangoon", "Asia/Yangon", "Asia/Barnaul", "Asia/Tomsk", "Indian/Christmas", "Antarctica/Davis", "Asia/Hovd", "Asia/Bangkok", "Asia/Ho_Chi_Minh", "Asia/Phnom_Penh", "Asia/Saigon", "Asia/Vientiane", "Asia/Krasnoyarsk", "Asia/Novokuznetsk", "Asia/Novosibirsk", "Asia/Jakarta", "Asia/Pontianak", "Antarctica/Casey", "Australia/Perth", "Australia/West", "Asia/Brunei", "Asia/Choibalsan", "Asia/Chongqing", "Asia/Chungking", "Asia/Harbin", "Asia/Macao", "Asia/Macau", "Asia/Shanghai", "Asia/Taipei", "PRC", "Asia/Hong_Kong", "Hongkong", "Asia/Irkutsk", "Asia/Kuala_Lumpur", "Asia/Kuching", "Asia/Manila", "Asia/Singapore", "Singapore", "Asia/Ulaanbaatar", "Asia/Ulan_Bator", "Asia/Makassar", "Asia/Ujung_Pandang", "Australia/Eucla", "Asia/Tokyo", "Japan", "Asia/Pyongyang", "Asia/Seoul", "ROK", "Pacific/Palau", "Asia/Dili", "Asia/Jayapura", "Asia/Chita", "Asia/Khandyga", "Asia/Yakutsk", "Australia/Adelaide", "Australia/Broken_Hill", "Australia/Darwin", "Australia/North", "Australia/South", "Australia/Yancowinna", "Australia/ACT", "Australia/Brisbane", "Australia/Canberra", "Australia/Currie", "Australia/Hobart", "Australia/Lindeman", "Australia/Melbourne", "Australia/NSW", "Australia/Queensland", "Australia/Sydney", "Australia/Tasmania", "Australia/Victoria", "Pacific/Chuuk", "Pacific/Truk", "Pacific/Yap", "Pacific/Guam", "Pacific/Saipan", "Antarctica/DumontDUrville", "Pacific/Port_Moresby", "Asia/Ust-Nera", "Asia/Vladivostok", "Australia/LHI", "Australia/Lord_Howe", "Pacific/Bougainville", "Pacific/Kosrae", "Asia/Magadan", "Antarctica/Macquarie", "Pacific/Noumea", "Pacific/Norfolk", "Pacific/Pohnpei", "Pacific/Ponape", "Asia/Sakhalin", "Pacific/Guadalcanal", "Asia/Srednekolymsk", "Pacific/Efate", "Asia/Anadyr", "Pacific/Fiji", "Pacific/Tarawa", "Kwajalein", "Pacific/Kwajalein", "Pacific/Majuro", "Pacific/Nauru", "Antarctica/McMurdo", "Antarctica/South_Pole", "NZ", "Pacific/Auckland", "Asia/Kamchatka", "Pacific/Funafuti", "Pacific/Wake", "Pacific/Wallis", "NZ-CHAT", "Pacific/Chatham", "Pacific/Enderbury", "Pacific/Fakaofo", "Pacific/Tongatapu", "Pacific/Apia", "Pacific/Kiritimati" ],
"configDefinitions" : null,
"modelType" : "VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.pipeline.config.TimeZoneChooserValues",
"filteringConfig" : ""
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 180,
"displayMode" : "BASIC",
"name" : "zonedDateTimeTargetTimeZone",
"type" : "MODEL",
"defaultValue" : null
}, {
"fieldName" : "otherZonedDateTimeFormat",
"description" : "",
"max" : 9223372036854775807,
"label" : "Other Zoned DateTime Format",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"zonedDateTimeFormat" : [ "OTHER" ],
"targetType" : [ "ZONED_DATETIME", "STRING" ]
},
"dependsOn" : "",
"group" : "TYPE_CONVERSION",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 70,
"displayMode" : "BASIC",
"name" : "otherZonedDateTimeFormat",
"type" : "STRING",
"defaultValue" : null
}, {
"fieldName" : "encoding",
"description" : "",
"max" : 9223372036854775807,
"label" : "CharSet",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"targetType" : [ "STRING" ]
},
"dependsOn" : "",
"group" : "TYPE_CONVERSION",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "UTF-8", "US-ASCII", "UTF-16", "ISO-8859-1", "IBM-500", "GBK", "Big5", "Big5-HKSCS", "CESU-8", "EUC-JP", "EUC-KR", "GB18030", "GB2312", "IBM-Thai", "IBM00858", "IBM01140", "IBM01141", "IBM01142", "IBM01143", "IBM01144", "IBM01145", "IBM01146", "IBM01147", "IBM01148", "IBM01149", "IBM037", "IBM1026", "IBM1047", "IBM273", "IBM277", "IBM278", "IBM280", "IBM284", "IBM285", "IBM290", "IBM297", "IBM420", "IBM424", "IBM437", "IBM500", "IBM775", "IBM850", "IBM852", "IBM855", "IBM857", "IBM860", "IBM861", "IBM862", "IBM863", "IBM864", "IBM865", "IBM866", "IBM868", "IBM869", "IBM870", "IBM871", "IBM918", "ISO-2022-CN", "ISO-2022-JP", "ISO-2022-JP-2", "ISO-2022-KR", "ISO-8859-13", "ISO-8859-15", "ISO-8859-2", "ISO-8859-3", "ISO-8859-4", "ISO-8859-5", "ISO-8859-6", "ISO-8859-7", "ISO-8859-8", "ISO-8859-9", "JIS_X0201", "JIS_X0212-1990", "KOI8-R", "KOI8-U", "Shift_JIS", "TIS-620", "UTF-16BE", "UTF-16LE", "UTF-32", "UTF-32BE", "UTF-32LE", "windows-1250", "windows-1251", "windows-1252", "windows-1253", "windows-1254", "windows-1255", "windows-1256", "windows-1257", "windows-1258", "windows-31j", "x-Big5-HKSCS-2001", "x-Big5-Solaris", "x-COMPOUND_TEXT", "x-euc-jp-linux", "x-EUC-TW", "x-eucJP-Open", "x-IBM1006", "x-IBM1025", "x-IBM1046", "x-IBM1097", "x-IBM1098", "x-IBM1112", "x-IBM1122", "x-IBM1123", "x-IBM1124", "x-IBM1166", "x-IBM1364", "x-IBM1381", "x-IBM1383", "x-IBM300", "x-IBM33722", "x-IBM737", "x-IBM833", "x-IBM834", "x-IBM856", "x-IBM874", "x-IBM875", "x-IBM921", "x-IBM922", "x-IBM930", "x-IBM933", "x-IBM935", "x-IBM937", "x-IBM939", "x-IBM942", "x-IBM942C", "x-IBM943", "x-IBM943C", "x-IBM948", "x-IBM949", "x-IBM949C", "x-IBM950", "x-IBM964", "x-IBM970", "x-ISCII91", "x-ISO-2022-CN-CNS", "x-ISO-2022-CN-GB", "x-iso-8859-11", "x-JIS0208", "x-JISAutoDetect", "x-Johab", "x-MacArabic", "x-MacCentralEurope", "x-MacCroatian", "x-MacCyrillic", "x-MacDingbat", "x-MacGreek", "x-MacHebrew", "x-MacIceland", "x-MacRoman", "x-MacRomania", "x-MacSymbol", "x-MacThai", "x-MacTurkish", "x-MacUkraine", "x-MS932_0213", "x-MS950-HKSCS", "x-MS950-HKSCS-XP", "x-mswin-936", "x-PCK", "x-SJIS_0213", "x-UTF-16LE-BOM", "X-UTF-32BE-BOM", "X-UTF-32LE-BOM", "x-windows-50220", "x-windows-50221", "x-windows-874", "x-windows-949", "x-windows-950", "x-windows-iso2022jp" ],
"values" : [ "UTF-8", "US-ASCII", "UTF-16", "ISO-8859-1", "IBM-500", "GBK", "Big5", "Big5-HKSCS", "CESU-8", "EUC-JP", "EUC-KR", "GB18030", "GB2312", "IBM-Thai", "IBM00858", "IBM01140", "IBM01141", "IBM01142", "IBM01143", "IBM01144", "IBM01145", "IBM01146", "IBM01147", "IBM01148", "IBM01149", "IBM037", "IBM1026", "IBM1047", "IBM273", "IBM277", "IBM278", "IBM280", "IBM284", "IBM285", "IBM290", "IBM297", "IBM420", "IBM424", "IBM437", "IBM500", "IBM775", "IBM850", "IBM852", "IBM855", "IBM857", "IBM860", "IBM861", "IBM862", "IBM863", "IBM864", "IBM865", "IBM866", "IBM868", "IBM869", "IBM870", "IBM871", "IBM918", "ISO-2022-CN", "ISO-2022-JP", "ISO-2022-JP-2", "ISO-2022-KR", "ISO-8859-13", "ISO-8859-15", "ISO-8859-2", "ISO-8859-3", "ISO-8859-4", "ISO-8859-5", "ISO-8859-6", "ISO-8859-7", "ISO-8859-8", "ISO-8859-9", "JIS_X0201", "JIS_X0212-1990", "KOI8-R", "KOI8-U", "Shift_JIS", "TIS-620", "UTF-16BE", "UTF-16LE", "UTF-32", "UTF-32BE", "UTF-32LE", "windows-1250", "windows-1251", "windows-1252", "windows-1253", "windows-1254", "windows-1255", "windows-1256", "windows-1257", "windows-1258", "windows-31j", "x-Big5-HKSCS-2001", "x-Big5-Solaris", "x-COMPOUND_TEXT", "x-euc-jp-linux", "x-EUC-TW", "x-eucJP-Open", "x-IBM1006", "x-IBM1025", "x-IBM1046", "x-IBM1097", "x-IBM1098", "x-IBM1112", "x-IBM1122", "x-IBM1123", "x-IBM1124", "x-IBM1166", "x-IBM1364", "x-IBM1381", "x-IBM1383", "x-IBM300", "x-IBM33722", "x-IBM737", "x-IBM833", "x-IBM834", "x-IBM856", "x-IBM874", "x-IBM875", "x-IBM921", "x-IBM922", "x-IBM930", "x-IBM933", "x-IBM935", "x-IBM937", "x-IBM939", "x-IBM942", "x-IBM942C", "x-IBM943", "x-IBM943C", "x-IBM948", "x-IBM949", "x-IBM949C", "x-IBM950", "x-IBM964", "x-IBM970", "x-ISCII91", "x-ISO-2022-CN-CNS", "x-ISO-2022-CN-GB", "x-iso-8859-11", "x-JIS0208", "x-JISAutoDetect", "x-Johab", "x-MacArabic", "x-MacCentralEurope", "x-MacCroatian", "x-MacCyrillic", "x-MacDingbat", "x-MacGreek", "x-MacHebrew", "x-MacIceland", "x-MacRoman", "x-MacRomania", "x-MacSymbol", "x-MacThai", "x-MacTurkish", "x-MacUkraine", "x-MS932_0213", "x-MS950-HKSCS", "x-MS950-HKSCS-XP", "x-mswin-936", "x-PCK", "x-SJIS_0213", "x-UTF-16LE-BOM", "X-UTF-32BE-BOM", "X-UTF-32LE-BOM", "x-windows-50220", "x-windows-50221", "x-windows-874", "x-windows-949", "x-windows-950", "x-windows-iso2022jp" ],
"configDefinitions" : null,
"modelType" : "VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.pipeline.config.CharsetChooserValues",
"filteringConfig" : ""
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 80,
"displayMode" : "BASIC",
"name" : "encoding",
"type" : "MODEL",
"defaultValue" : "UTF-8"
} ],
"modelType" : "LIST_BEAN",
"valuesProviderClass" : null,
"filteringConfig" : null
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 10,
"displayMode" : "BASIC",
"name" : "wholeTypeConverterConfigs",
"type" : "MODEL",
"defaultValue" : [ ]
}, {
"fieldName" : "stageOnRecordError",
"description" : "Action to take with records sent to error",
"max" : 9223372036854775807,
"label" : "On Record Error",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : { },
"dependsOn" : "",
"group" : "",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "Discard", "Send to Error", "Stop Pipeline" ],
"values" : [ "DISCARD", "TO_ERROR", "STOP_PIPELINE" ],
"configDefinitions" : null,
"modelType" : "VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.datacollector.config.OnRecordErrorChooserValues",
"filteringConfig" : ""
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 30,
"displayMode" : "BASIC",
"name" : "stageOnRecordError",
"type" : "MODEL",
"defaultValue" : "TO_ERROR"
}, {
"fieldName" : "stageRequiredFields",
"description" : "Records without any of these fields are sent to error",
"max" : 9223372036854775807,
"label" : "Required Fields",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : { },
"dependsOn" : "",
"group" : "",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : null,
"values" : null,
"configDefinitions" : null,
"modelType" : "FIELD_SELECTOR_MULTI_VALUE",
"valuesProviderClass" : null,
"filteringConfig" : null
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 10,
"displayMode" : "BASIC",
"name" : "stageRequiredFields",
"type" : "MODEL",
"defaultValue" : [ ]
}, {
"fieldName" : "stageRecordPreconditions",
"description" : "Records that don't satisfy all the preconditions are sent to error",
"max" : 9223372036854775807,
"label" : "Preconditions",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : { },
"dependsOn" : "",
"group" : "",
"evaluation" : "EXPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61", "67", "68", "69", "70", "71", "72", "73", "74", "75", "76", "77", "78", "79", "80", "81", "82", "83", "84", "85", "86", "87", "88", "89", "90", "91", "92", "93", "94", "95" ],
"elConstantDefinitionsIdx" : [ "62", "96", "97", "98", "99", "100", "101", "102", "103", "104", "105", "106", "107", "108", "109", "110", "111", "112" ],
"displayPosition" : 20,
"displayMode" : "BASIC",
"name" : "stageRecordPreconditions",
"type" : "LIST",
"defaultValue" : null
} ],
"configGroupDefinition" : {
"classNameToGroupsMap" : {
"com.streamsets.pipeline.stage.processor.fieldtypeconverter.Groups" : [ "TYPE_CONVERSION" ]
},
"groupNameToLabelMapList" : [ {
"name" : "TYPE_CONVERSION",
"label" : "Conversions"
} ]
},
"libraryLabel" : "Basic",
"pipelineLifecycleStage" : false,
"statsAggregatorStage" : false,
"errorStage" : false,
"rawSourceDefinition" : null,
"offsetCommitTrigger" : false,
"variableOutputStreams" : false,
"outputStreams" : 1,
"outputStreamLabelProviderClass" : "com.streamsets.pipeline.api.StageDef$DefaultOutputStreams",
"outputStreamLabels" : null,
"executionModes" : [ "STANDALONE", "CLUSTER_BATCH", "CLUSTER_YARN_STREAMING", "CLUSTER_MESOS_STREAMING", "EMR_BATCH" ],
"resetOffset" : false,
"onlineHelpRefUrl" : "index.html?contextID=task_g23_2tq_wq",
"hideStage" : [ ],
"outputStreamsDrivenByConfig" : "",
"sendsResponse" : false,
"beta" : false,
"inputStreams" : -1,
"inputStreamLabelProviderClass" : "com.streamsets.pipeline.api.StageDef$DefaultInputStreams",
"inputStreamLabels" : null,
"bisectable" : false,
"yamlUpgrader" : "upgrader/FieldTypeConverterDProcessor.yaml",
"name" : "com_streamsets_pipeline_stage_processor_fieldtypeconverter_FieldTypeConverterDProcessor",
"type" : "PROCESSOR",
"className" : "com.streamsets.pipeline.stage.processor.fieldtypeconverter.FieldTypeConverterDProcessor",
"preconditions" : true,
"onRecordError" : true,
"producingEvents" : false
}, {
"icon" : "expression.png",
"version" : "2",
"privateClassLoader" : false,
"description" : "Performs calculations on a field-by-field basis",
"services" : [ ],
"eventDefs" : [ ],
"library" : "streamsets-datacollector-basic-lib",
"label" : "Expression Evaluator",
"configDefinitions" : [ {
"fieldName" : "expressionProcessorConfigs",
"description" : "",
"max" : 9223372036854775807,
"label" : "Field Expressions",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : { },
"dependsOn" : "",
"group" : "EXPRESSIONS",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : null,
"values" : null,
"configDefinitions" : [ {
"fieldName" : "fieldToSet",
"description" : "Use an existing field or enter a new field. Using an existing field overwrites the original value.",
"max" : 9223372036854775807,
"label" : "Output Field",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : { },
"dependsOn" : "",
"group" : "EXPRESSIONS",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : null,
"values" : null,
"configDefinitions" : null,
"modelType" : "FIELD_SELECTOR",
"valuesProviderClass" : null,
"filteringConfig" : null
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 10,
"displayMode" : "BASIC",
"name" : "fieldToSet",
"type" : "MODEL",
"defaultValue" : "/"
}, {
"fieldName" : "expression",
"description" : "Use the expression language to modify values in a field.",
"max" : 9223372036854775807,
"label" : "Field Expression",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : { },
"dependsOn" : "",
"group" : "EXPRESSIONS",
"evaluation" : "EXPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61", "116", "117", "118", "119", "120", "121", "122", "123", "124", "125", "126", "127", "128", "129", "130", "131", "132", "133", "134", "135", "136", "137", "138", "139", "140", "141", "142", "143", "144", "197", "198", "199", "200", "201", "202", "168", "169", "170", "171", "172", "173", "174", "175", "176", "177", "178", "179", "180" ],
"elConstantDefinitionsIdx" : [ "62", "151", "152", "153", "154", "155", "156", "157", "158", "159", "160", "161", "162", "163", "164", "165", "166", "167" ],
"displayPosition" : 20,
"displayMode" : "BASIC",
"name" : "expression",
"type" : "STRING",
"defaultValue" : "${record:value('/')}"
} ],
"modelType" : "LIST_BEAN",
"valuesProviderClass" : null,
"filteringConfig" : null
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 10,
"displayMode" : "BASIC",
"name" : "expressionProcessorConfigs",
"type" : "MODEL",
"defaultValue" : [ ]
}, {
"fieldName" : "headerAttributeConfigs",
"description" : "",
"max" : 9223372036854775807,
"label" : "Header Attribute Expressions",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : { },
"dependsOn" : "",
"group" : "EXPRESSIONS",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : null,
"values" : null,
"configDefinitions" : [ {
"fieldName" : "attributeToSet",
"description" : "Use an existing header attribute or enter a new attribute. Using an existing attribute overwrites the original value.",
"max" : 9223372036854775807,
"label" : "Header Attribute",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : { },
"dependsOn" : "",
"group" : "EXPRESSIONS",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 30,
"displayMode" : "BASIC",
"name" : "attributeToSet",
"type" : "STRING",
"defaultValue" : null
}, {
"fieldName" : "headerAttributeExpression",
"description" : "Use the expression language to modify or set new attributes in the header.",
"max" : 9223372036854775807,
"label" : "Header Attribute Expression",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : { },
"dependsOn" : "",
"group" : "EXPRESSIONS",
"evaluation" : "EXPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61", "116", "117", "118", "119", "120", "121", "122", "123", "124", "125", "126", "127", "128", "129", "130", "131", "132", "133", "134", "135", "136", "137", "138", "139", "140", "141", "142", "143", "144", "168", "169", "170", "171", "172", "173", "174", "175", "176", "177", "178", "179", "180", "197", "198", "199", "200", "201", "202" ],
"elConstantDefinitionsIdx" : [ "62", "151", "152", "153", "154", "155", "156", "157", "158", "159", "160", "161", "162", "163", "164", "165", "166", "167" ],
"displayPosition" : 40,
"displayMode" : "BASIC",
"name" : "headerAttributeExpression",
"type" : "STRING",
"defaultValue" : null
} ],
"modelType" : "LIST_BEAN",
"valuesProviderClass" : null,
"filteringConfig" : null
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 20,
"displayMode" : "BASIC",
"name" : "headerAttributeConfigs",
"type" : "MODEL",
"defaultValue" : [ ]
}, {
"fieldName" : "fieldAttributeConfigs",
"description" : "",
"max" : 9223372036854775807,
"label" : "Field Attribute Expressions",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : { },
"dependsOn" : "",
"group" : "EXPRESSIONS",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : null,
"values" : null,
"configDefinitions" : [ {
"fieldName" : "fieldToSet",
"description" : "The existing field which will receive the attribute value.",
"max" : 9223372036854775807,
"label" : "Field",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : { },
"dependsOn" : "",
"group" : "EXPRESSIONS",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : null,
"values" : null,
"configDefinitions" : null,
"modelType" : "FIELD_SELECTOR",
"valuesProviderClass" : null,
"filteringConfig" : null
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 10,
"displayMode" : "BASIC",
"name" : "fieldToSet",
"type" : "MODEL",
"defaultValue" : "/"
}, {
"fieldName" : "attributeToSet",
"description" : "Use an existing field attribute or enter a new attribute. Using an existing attribute overwrites the original value.",
"max" : 9223372036854775807,
"label" : "Attribute Name",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : { },
"dependsOn" : "",
"group" : "EXPRESSIONS",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 30,
"displayMode" : "BASIC",
"name" : "attributeToSet",
"type" : "STRING",
"defaultValue" : null
}, {
"fieldName" : "fieldAttributeExpression",
"description" : "Use the expression language to modify or set new attributes in the field.",
"max" : 9223372036854775807,
"label" : "Header Attribute Expression",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : { },
"dependsOn" : "",
"group" : "EXPRESSIONS",
"evaluation" : "EXPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61", "116", "117", "118", "119", "120", "121", "122", "123", "124", "125", "126", "127", "128", "129", "130", "131", "132", "133", "134", "135", "136", "137", "138", "139", "140", "141", "142", "143", "144", "197", "198", "199", "200", "201", "202" ],
"elConstantDefinitionsIdx" : [ "62", "151", "152", "153", "154", "155", "156", "157", "158", "159", "160", "161", "162", "163", "164", "165", "166", "167" ],
"displayPosition" : 50,
"displayMode" : "BASIC",
"name" : "fieldAttributeExpression",
"type" : "STRING",
"defaultValue" : null
} ],
"modelType" : "LIST_BEAN",
"valuesProviderClass" : null,
"filteringConfig" : null
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 30,
"displayMode" : "BASIC",
"name" : "fieldAttributeConfigs",
"type" : "MODEL",
"defaultValue" : [ ]
}, {
"fieldName" : "stageOnRecordError",
"description" : "Action to take with records sent to error",
"max" : 9223372036854775807,
"label" : "On Record Error",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : { },
"dependsOn" : "",
"group" : "",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "Discard", "Send to Error", "Stop Pipeline" ],
"values" : [ "DISCARD", "TO_ERROR", "STOP_PIPELINE" ],
"configDefinitions" : null,
"modelType" : "VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.datacollector.config.OnRecordErrorChooserValues",
"filteringConfig" : ""
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 30,
"displayMode" : "BASIC",
"name" : "stageOnRecordError",
"type" : "MODEL",
"defaultValue" : "TO_ERROR"
}, {
"fieldName" : "stageRequiredFields",
"description" : "Records without any of these fields are sent to error",
"max" : 9223372036854775807,
"label" : "Required Fields",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : { },
"dependsOn" : "",
"group" : "",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : null,
"values" : null,
"configDefinitions" : null,
"modelType" : "FIELD_SELECTOR_MULTI_VALUE",
"valuesProviderClass" : null,
"filteringConfig" : null
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 10,
"displayMode" : "BASIC",
"name" : "stageRequiredFields",
"type" : "MODEL",
"defaultValue" : [ ]
}, {
"fieldName" : "stageRecordPreconditions",
"description" : "Records that don't satisfy all the preconditions are sent to error",
"max" : 9223372036854775807,
"label" : "Preconditions",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : { },
"dependsOn" : "",
"group" : "",
"evaluation" : "EXPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61", "67", "68", "69", "70", "71", "72", "73", "74", "75", "76", "77", "78", "79", "80", "81", "82", "83", "84", "85", "86", "87", "88", "89", "90", "91", "92", "93", "94", "95" ],
"elConstantDefinitionsIdx" : [ "62", "96", "97", "98", "99", "100", "101", "102", "103", "104", "105", "106", "107", "108", "109", "110", "111", "112" ],
"displayPosition" : 20,
"displayMode" : "BASIC",
"name" : "stageRecordPreconditions",
"type" : "LIST",
"defaultValue" : null
} ],
"configGroupDefinition" : {
"classNameToGroupsMap" : {
"com.streamsets.pipeline.stage.processor.expression.Groups" : [ "EXPRESSIONS" ]
},
"groupNameToLabelMapList" : [ {
"name" : "EXPRESSIONS",
"label" : "Expressions"
} ]
},
"libraryLabel" : "Basic",
"pipelineLifecycleStage" : false,
"statsAggregatorStage" : false,
"errorStage" : false,
"rawSourceDefinition" : null,
"offsetCommitTrigger" : false,
"variableOutputStreams" : false,
"outputStreams" : 1,
"outputStreamLabelProviderClass" : "com.streamsets.pipeline.api.StageDef$DefaultOutputStreams",
"outputStreamLabels" : null,
"executionModes" : [ "STANDALONE", "CLUSTER_BATCH", "CLUSTER_YARN_STREAMING", "CLUSTER_MESOS_STREAMING", "EDGE", "EMR_BATCH" ],
"resetOffset" : false,
"onlineHelpRefUrl" : "index.html?contextID=task_x2h_tv4_yq",
"hideStage" : [ ],
"outputStreamsDrivenByConfig" : "",
"sendsResponse" : false,
"beta" : false,
"inputStreams" : -1,
"inputStreamLabelProviderClass" : "com.streamsets.pipeline.api.StageDef$DefaultInputStreams",
"inputStreamLabels" : null,
"bisectable" : false,
"yamlUpgrader" : "upgrader/ExpressionDProcessor.yaml",
"name" : "com_streamsets_pipeline_stage_processor_expression_ExpressionDProcessor",
"type" : "PROCESSOR",
"className" : "com.streamsets.pipeline.stage.processor.expression.ExpressionDProcessor",
"preconditions" : true,
"onRecordError" : true,
"producingEvents" : false
}, {
"icon" : "trash.png",
"version" : "1",
"privateClassLoader" : false,
"description" : "Discards records",
"services" : [ ],
"eventDefs" : [ ],
"library" : "streamsets-datacollector-basic-lib",
"label" : "Trash",
"configDefinitions" : [ ],
"configGroupDefinition" : {
"classNameToGroupsMap" : { },
"groupNameToLabelMapList" : [ ]
},
"libraryLabel" : "Basic",
"pipelineLifecycleStage" : false,
"statsAggregatorStage" : false,
"errorStage" : false,
"rawSourceDefinition" : null,
"offsetCommitTrigger" : false,
"variableOutputStreams" : false,
"outputStreams" : 0,
"outputStreamLabelProviderClass" : null,
"outputStreamLabels" : null,
"executionModes" : [ "STANDALONE", "CLUSTER_BATCH", "CLUSTER_YARN_STREAMING", "CLUSTER_MESOS_STREAMING", "EDGE", "EMR_BATCH" ],
"resetOffset" : false,
"onlineHelpRefUrl" : "index.html?contextID=task_ad4_qyl_zq",
"hideStage" : [ ],
"outputStreamsDrivenByConfig" : "",
"sendsResponse" : false,
"beta" : false,
"inputStreams" : -1,
"inputStreamLabelProviderClass" : "com.streamsets.pipeline.api.StageDef$DefaultInputStreams",
"inputStreamLabels" : null,
"bisectable" : false,
"yamlUpgrader" : "upgrader/NullDTarget.yaml",
"name" : "com_streamsets_pipeline_stage_destination_devnull_NullDTarget",
"type" : "TARGET",
"className" : "com.streamsets.pipeline.stage.destination.devnull.NullDTarget",
"preconditions" : false,
"onRecordError" : false,
"producingEvents" : false
}, {
"icon" : "",
"version" : "1",
"privateClassLoader" : false,
"description" : "Discards records and events",
"services" : [ ],
"eventDefs" : [ ],
"library" : "streamsets-datacollector-basic-lib",
"label" : "Discard",
"configDefinitions" : [ ],
"configGroupDefinition" : {
"classNameToGroupsMap" : { },
"groupNameToLabelMapList" : [ ]
},
"libraryLabel" : "Basic",
"pipelineLifecycleStage" : true,
"statsAggregatorStage" : false,
"errorStage" : true,
"rawSourceDefinition" : null,
"offsetCommitTrigger" : false,
"variableOutputStreams" : false,
"outputStreams" : 0,
"outputStreamLabelProviderClass" : null,
"outputStreamLabels" : null,
"executionModes" : [ "STANDALONE", "CLUSTER_BATCH", "CLUSTER_YARN_STREAMING", "CLUSTER_MESOS_STREAMING", "EDGE", "EMR_BATCH" ],
"resetOffset" : false,
"onlineHelpRefUrl" : "index.html?contextID=concept_kgc_l4y_5r",
"hideStage" : [ "ERROR_STAGE", "LIFECYCLE_STAGE" ],
"outputStreamsDrivenByConfig" : "",
"sendsResponse" : false,
"beta" : false,
"inputStreams" : -1,
"inputStreamLabelProviderClass" : "com.streamsets.pipeline.api.StageDef$DefaultInputStreams",
"inputStreamLabels" : null,
"bisectable" : false,
"yamlUpgrader" : "upgrader/ToErrorNullDTarget.yaml",
"name" : "com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget",
"type" : "TARGET",
"className" : "com.streamsets.pipeline.stage.destination.devnull.ToErrorNullDTarget",
"preconditions" : false,
"onRecordError" : false,
"producingEvents" : false
}, {
"icon" : "dev.png",
"version" : "3",
"privateClassLoader" : false,
"description" : "Add Raw data to the source.",
"services" : [ {
"service" : "com.streamsets.pipeline.api.service.dataformats.DataFormatParserService",
"configuration" : {
"displayFormats" : "DELIMITED,JSON,LOG,SDC_JSON,TEXT,XML",
"dataFormat" : "JSON"
}
} ],
"eventDefs" : [ ],
"library" : "streamsets-datacollector-dev-lib",
"label" : "Dev Raw Data Source",
"configDefinitions" : [ {
"fieldName" : "rawData",
"description" : "",
"max" : 9223372036854775807,
"label" : "Raw Data",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : { },
"dependsOn" : "",
"group" : "RAW",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "application/json",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 1,
"displayMode" : "BASIC",
"name" : "rawData",
"type" : "TEXT",
"defaultValue" : "{\n \"f1\": \"abc\",\n \"f2\": \"xyz\",\n \"f3\": \"lmn\"\n}"
}, {
"fieldName" : "stopAfterFirstBatch",
"description" : "",
"max" : 9223372036854775807,
"label" : "Stop After First Batch",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : { },
"dependsOn" : "",
"group" : "RAW",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 2,
"displayMode" : "BASIC",
"name" : "stopAfterFirstBatch",
"type" : "BOOLEAN",
"defaultValue" : false
}, {
"fieldName" : "eventData",
"description" : "",
"max" : 9223372036854775807,
"label" : "Event Data",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : { },
"dependsOn" : "",
"group" : "EVENT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "application/json",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 20,
"displayMode" : "BASIC",
"name" : "eventData",
"type" : "TEXT",
"defaultValue" : null
}, {
"fieldName" : "stageOnRecordError",
"description" : "Action to take with records sent to error",
"max" : 9223372036854775807,
"label" : "On Record Error",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : { },
"dependsOn" : "",
"group" : "",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "Discard", "Send to Error", "Stop Pipeline" ],
"values" : [ "DISCARD", "TO_ERROR", "STOP_PIPELINE" ],
"configDefinitions" : null,
"modelType" : "VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.datacollector.config.OnRecordErrorChooserValues",
"filteringConfig" : ""
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 30,
"displayMode" : "BASIC",
"name" : "stageOnRecordError",
"type" : "MODEL",
"defaultValue" : "TO_ERROR"
} ],
"configGroupDefinition" : {
"classNameToGroupsMap" : {
"com.streamsets.pipeline.stage.devtest.rawdata.RawDataSourceGroups" : [ "RAW", "EVENT" ]
},
"groupNameToLabelMapList" : [ {
"name" : "RAW",
"label" : "Raw Data"
}, {
"name" : "EVENT",
"label" : "Event Data"
} ]
},
"libraryLabel" : "Dev (for development only)",
"pipelineLifecycleStage" : false,
"statsAggregatorStage" : false,
"errorStage" : false,
"rawSourceDefinition" : null,
"offsetCommitTrigger" : false,
"variableOutputStreams" : false,
"outputStreams" : 1,
"outputStreamLabelProviderClass" : "com.streamsets.pipeline.api.StageDef$DefaultOutputStreams",
"outputStreamLabels" : null,
"executionModes" : [ "STANDALONE", "EDGE" ],
"resetOffset" : false,
"onlineHelpRefUrl" : "index.html#datacollector/UserGuide/Pipeline_Design/DevStages.html",
"hideStage" : [ ],
"outputStreamsDrivenByConfig" : "",
"sendsResponse" : false,
"beta" : false,
"inputStreams" : -1,
"inputStreamLabelProviderClass" : "com.streamsets.pipeline.api.StageDef$DefaultInputStreams",
"inputStreamLabels" : null,
"bisectable" : false,
"yamlUpgrader" : "upgrader/RawDataDSource.yaml",
"name" : "com_streamsets_pipeline_stage_devtest_rawdata_RawDataDSource",
"type" : "SOURCE",
"className" : "com.streamsets.pipeline.stage.devtest.rawdata.RawDataDSource",
"preconditions" : false,
"onRecordError" : true,
"producingEvents" : true
}, {
"icon" : "",
"version" : "1",
"privateClassLoader" : false,
"description" : "Writes pipeline statistics to Control Hub directly without any aggregation",
"services" : [ ],
"eventDefs" : [ ],
"library" : "streamsets-datacollector-basic-lib",
"label" : "Write Directly to Control Hub - statistics are not aggregated across Data Collectors",
"configDefinitions" : [ ],
"configGroupDefinition" : {
"classNameToGroupsMap" : { },
"groupNameToLabelMapList" : [ ]
},
"libraryLabel" : "Basic",
"pipelineLifecycleStage" : false,
"statsAggregatorStage" : true,
"errorStage" : false,
"rawSourceDefinition" : null,
"offsetCommitTrigger" : false,
"variableOutputStreams" : false,
"outputStreams" : 0,
"outputStreamLabelProviderClass" : null,
"outputStreamLabels" : null,
"executionModes" : [ "STANDALONE", "CLUSTER_BATCH", "CLUSTER_YARN_STREAMING", "CLUSTER_MESOS_STREAMING", "EMR_BATCH", "EDGE" ],
"resetOffset" : false,
"onlineHelpRefUrl" : "",
"hideStage" : [ "STATS_AGGREGATOR_STAGE" ],
"outputStreamsDrivenByConfig" : "",
"sendsResponse" : false,
"beta" : false,
"inputStreams" : -1,
"inputStreamLabelProviderClass" : "com.streamsets.pipeline.api.StageDef$DefaultInputStreams",
"inputStreamLabels" : null,
"bisectable" : false,
"yamlUpgrader" : "upgrader/StatsDpmDirectlyDTarget.yaml",
"name" : "com_streamsets_pipeline_stage_destination_devnull_StatsDpmDirectlyDTarget",
"type" : "TARGET",
"className" : "com.streamsets.pipeline.stage.destination.devnull.StatsDpmDirectlyDTarget",
"preconditions" : false,
"onRecordError" : false,
"producingEvents" : false
} ],
"services" : [ {
"version" : "1",
"privateClassLoader" : false,
"description" : "",
"provides" : "com.streamsets.pipeline.api.service.dataformats.DataFormatParserService",
"library" : "streamsets-datacollector-dataformats-lib",
"label" : "DataFormat Parser",
"configDefinitions" : [ {
"fieldName" : "displayFormats",
"description" : "",
"max" : 9223372036854775807,
"label" : "List of formats that should be displayed to the user.",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : { },
"dependsOn" : "",
"group" : "",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 0,
"displayMode" : "BASIC",
"name" : "displayFormats",
"type" : "RUNTIME",
"defaultValue" : null
}, {
"fieldName" : "dataFormat",
"description" : "",
"max" : 9223372036854775807,
"label" : "Data Format",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : { },
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "Avro", "Binary", "Datagram", "Delimited", "Excel", "JSON", "Log", "Netflow", "Protobuf", "SDC Record", "Syslog", "Text", "Whole File", "XML" ],
"values" : [ "AVRO", "BINARY", "DATAGRAM", "DELIMITED", "EXCEL", "JSON", "LOG", "NETFLOW", "PROTOBUF", "SDC_JSON", "SYSLOG", "TEXT", "WHOLE_FILE", "XML" ],
"configDefinitions" : null,
"modelType" : "VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.service.parser.DataFormatChooserValues",
"filteringConfig" : "displayFormats"
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 1,
"displayMode" : "BASIC",
"name" : "dataFormat",
"type" : "MODEL",
"defaultValue" : null
}, {
"fieldName" : "compression",
"description" : "Compression formats gzip, bzip2, xz, lzma, Pack200, DEFLATE and Z are supported. Archive formats 7z, ar, arj, cpio, dump, tar and zip are supported.",
"max" : 9223372036854775807,
"label" : "Compression Format",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormat" : [ "SDC_JSON", "DELIMITED", "TEXT", "XML", "BINARY", "LOG", "PROTOBUF", "JSON" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "None", "Compressed File", "Archive", "Compressed Archive" ],
"values" : [ "NONE", "COMPRESSED_FILE", "ARCHIVE", "COMPRESSED_ARCHIVE" ],
"configDefinitions" : null,
"modelType" : "VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.pipeline.config.CompressionChooserValues",
"filteringConfig" : ""
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 2,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.compression",
"type" : "MODEL",
"defaultValue" : "NONE"
}, {
"fieldName" : "filePatternInArchive",
"description" : "A glob pattern that defines the pattern of the file names within the compressed directory.",
"max" : 9223372036854775807,
"label" : "File Name Pattern within Compressed Directory",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormat" : [ "SDC_JSON", "DELIMITED", "TEXT", "XML", "BINARY", "LOG", "PROTOBUF", "JSON" ],
"dataFormatConfig.compression" : [ "COMPRESSED_ARCHIVE", "ARCHIVE" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 3,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.filePatternInArchive",
"type" : "STRING",
"defaultValue" : "*"
}, {
"fieldName" : "charset",
"description" : "",
"max" : 9223372036854775807,
"label" : "Charset",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormat" : [ "DELIMITED", "TEXT", "XML", "LOG", "DATAGRAM", "JSON" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "UTF-8", "US-ASCII", "UTF-16", "ISO-8859-1", "IBM-500", "GBK", "Big5", "Big5-HKSCS", "CESU-8", "EUC-JP", "EUC-KR", "GB18030", "GB2312", "IBM-Thai", "IBM00858", "IBM01140", "IBM01141", "IBM01142", "IBM01143", "IBM01144", "IBM01145", "IBM01146", "IBM01147", "IBM01148", "IBM01149", "IBM037", "IBM1026", "IBM1047", "IBM273", "IBM277", "IBM278", "IBM280", "IBM284", "IBM285", "IBM290", "IBM297", "IBM420", "IBM424", "IBM437", "IBM500", "IBM775", "IBM850", "IBM852", "IBM855", "IBM857", "IBM860", "IBM861", "IBM862", "IBM863", "IBM864", "IBM865", "IBM866", "IBM868", "IBM869", "IBM870", "IBM871", "IBM918", "ISO-2022-CN", "ISO-2022-JP", "ISO-2022-JP-2", "ISO-2022-KR", "ISO-8859-13", "ISO-8859-15", "ISO-8859-2", "ISO-8859-3", "ISO-8859-4", "ISO-8859-5", "ISO-8859-6", "ISO-8859-7", "ISO-8859-8", "ISO-8859-9", "JIS_X0201", "JIS_X0212-1990", "KOI8-R", "KOI8-U", "Shift_JIS", "TIS-620", "UTF-16BE", "UTF-16LE", "UTF-32", "UTF-32BE", "UTF-32LE", "windows-1250", "windows-1251", "windows-1252", "windows-1253", "windows-1254", "windows-1255", "windows-1256", "windows-1257", "windows-1258", "windows-31j", "x-Big5-HKSCS-2001", "x-Big5-Solaris", "x-COMPOUND_TEXT", "x-euc-jp-linux", "x-EUC-TW", "x-eucJP-Open", "x-IBM1006", "x-IBM1025", "x-IBM1046", "x-IBM1097", "x-IBM1098", "x-IBM1112", "x-IBM1122", "x-IBM1123", "x-IBM1124", "x-IBM1166", "x-IBM1364", "x-IBM1381", "x-IBM1383", "x-IBM300", "x-IBM33722", "x-IBM737", "x-IBM833", "x-IBM834", "x-IBM856", "x-IBM874", "x-IBM875", "x-IBM921", "x-IBM922", "x-IBM930", "x-IBM933", "x-IBM935", "x-IBM937", "x-IBM939", "x-IBM942", "x-IBM942C", "x-IBM943", "x-IBM943C", "x-IBM948", "x-IBM949", "x-IBM949C", "x-IBM950", "x-IBM964", "x-IBM970", "x-ISCII91", "x-ISO-2022-CN-CNS", "x-ISO-2022-CN-GB", "x-iso-8859-11", "x-JIS0208", "x-JISAutoDetect", "x-Johab", "x-MacArabic", "x-MacCentralEurope", "x-MacCroatian", "x-MacCyrillic", "x-MacDingbat", "x-MacGreek", "x-MacHebrew", "x-MacIceland", "x-MacRoman", "x-MacRomania", "x-MacSymbol", "x-MacThai", "x-MacTurkish", "x-MacUkraine", "x-MS932_0213", "x-MS950-HKSCS", "x-MS950-HKSCS-XP", "x-mswin-936", "x-PCK", "x-SJIS_0213", "x-UTF-16LE-BOM", "X-UTF-32BE-BOM", "X-UTF-32LE-BOM", "x-windows-50220", "x-windows-50221", "x-windows-874", "x-windows-949", "x-windows-950", "x-windows-iso2022jp" ],
"values" : [ "UTF-8", "US-ASCII", "UTF-16", "ISO-8859-1", "IBM-500", "GBK", "Big5", "Big5-HKSCS", "CESU-8", "EUC-JP", "EUC-KR", "GB18030", "GB2312", "IBM-Thai", "IBM00858", "IBM01140", "IBM01141", "IBM01142", "IBM01143", "IBM01144", "IBM01145", "IBM01146", "IBM01147", "IBM01148", "IBM01149", "IBM037", "IBM1026", "IBM1047", "IBM273", "IBM277", "IBM278", "IBM280", "IBM284", "IBM285", "IBM290", "IBM297", "IBM420", "IBM424", "IBM437", "IBM500", "IBM775", "IBM850", "IBM852", "IBM855", "IBM857", "IBM860", "IBM861", "IBM862", "IBM863", "IBM864", "IBM865", "IBM866", "IBM868", "IBM869", "IBM870", "IBM871", "IBM918", "ISO-2022-CN", "ISO-2022-JP", "ISO-2022-JP-2", "ISO-2022-KR", "ISO-8859-13", "ISO-8859-15", "ISO-8859-2", "ISO-8859-3", "ISO-8859-4", "ISO-8859-5", "ISO-8859-6", "ISO-8859-7", "ISO-8859-8", "ISO-8859-9", "JIS_X0201", "JIS_X0212-1990", "KOI8-R", "KOI8-U", "Shift_JIS", "TIS-620", "UTF-16BE", "UTF-16LE", "UTF-32", "UTF-32BE", "UTF-32LE", "windows-1250", "windows-1251", "windows-1252", "windows-1253", "windows-1254", "windows-1255", "windows-1256", "windows-1257", "windows-1258", "windows-31j", "x-Big5-HKSCS-2001", "x-Big5-Solaris", "x-COMPOUND_TEXT", "x-euc-jp-linux", "x-EUC-TW", "x-eucJP-Open", "x-IBM1006", "x-IBM1025", "x-IBM1046", "x-IBM1097", "x-IBM1098", "x-IBM1112", "x-IBM1122", "x-IBM1123", "x-IBM1124", "x-IBM1166", "x-IBM1364", "x-IBM1381", "x-IBM1383", "x-IBM300", "x-IBM33722", "x-IBM737", "x-IBM833", "x-IBM834", "x-IBM856", "x-IBM874", "x-IBM875", "x-IBM921", "x-IBM922", "x-IBM930", "x-IBM933", "x-IBM935", "x-IBM937", "x-IBM939", "x-IBM942", "x-IBM942C", "x-IBM943", "x-IBM943C", "x-IBM948", "x-IBM949", "x-IBM949C", "x-IBM950", "x-IBM964", "x-IBM970", "x-ISCII91", "x-ISO-2022-CN-CNS", "x-ISO-2022-CN-GB", "x-iso-8859-11", "x-JIS0208", "x-JISAutoDetect", "x-Johab", "x-MacArabic", "x-MacCentralEurope", "x-MacCroatian", "x-MacCyrillic", "x-MacDingbat", "x-MacGreek", "x-MacHebrew", "x-MacIceland", "x-MacRoman", "x-MacRomania", "x-MacSymbol", "x-MacThai", "x-MacTurkish", "x-MacUkraine", "x-MS932_0213", "x-MS950-HKSCS", "x-MS950-HKSCS-XP", "x-mswin-936", "x-PCK", "x-SJIS_0213", "x-UTF-16LE-BOM", "X-UTF-32BE-BOM", "X-UTF-32LE-BOM", "x-windows-50220", "x-windows-50221", "x-windows-874", "x-windows-949", "x-windows-950", "x-windows-iso2022jp" ],
"configDefinitions" : null,
"modelType" : "VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.pipeline.config.CharsetChooserValues",
"filteringConfig" : ""
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 999,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.charset",
"type" : "MODEL",
"defaultValue" : "UTF-8"
}, {
"fieldName" : "removeCtrlChars",
"description" : "Use only if required as it impacts reading performance",
"max" : 9223372036854775807,
"label" : "Ignore Control Characters",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormat" : [ "DELIMITED", "TEXT", "XML", "LOG", "DATAGRAM", "JSON" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 1000,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.removeCtrlChars",
"type" : "BOOLEAN",
"defaultValue" : false
}, {
"fieldName" : "textMaxLineLen",
"description" : "Longer lines are truncated",
"max" : 2147483647,
"label" : "Max Line Length",
"required" : true,
"min" : 1,
"dependsOnMap" : {
"dataFormat" : [ "TEXT" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 340,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.textMaxLineLen",
"type" : "NUMBER",
"defaultValue" : 1024
}, {
"fieldName" : "useCustomDelimiter",
"description" : "Use custom delimiters to create records",
"max" : 9223372036854775807,
"label" : "Use Custom Delimiter",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormat" : [ "TEXT" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 342,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.useCustomDelimiter",
"type" : "BOOLEAN",
"defaultValue" : false
}, {
"fieldName" : "customDelimiter",
"description" : "One or more characters. Leading and trailing spaces are stripped.",
"max" : 9223372036854775807,
"label" : "Custom Delimiter",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormat" : [ "TEXT" ],
"dataFormatConfig.useCustomDelimiter" : [ true ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 344,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.customDelimiter",
"type" : "STRING",
"defaultValue" : "\\r\\n"
}, {
"fieldName" : "includeCustomDelimiterInTheText",
"description" : "Include custom delimiters in the data",
"max" : 9223372036854775807,
"label" : "Include Custom Delimiter",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormat" : [ "TEXT" ],
"dataFormatConfig.useCustomDelimiter" : [ true ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 346,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.includeCustomDelimiterInTheText",
"type" : "BOOLEAN",
"defaultValue" : false
}, {
"fieldName" : "jsonContent",
"description" : "",
"max" : 9223372036854775807,
"label" : "JSON Content",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormat" : [ "JSON" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "JSON array of objects", "Multiple JSON objects" ],
"values" : [ "ARRAY_OBJECTS", "MULTIPLE_OBJECTS" ],
"configDefinitions" : null,
"modelType" : "VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.pipeline.config.JsonModeChooserValues",
"filteringConfig" : ""
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 350,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.jsonContent",
"type" : "MODEL",
"defaultValue" : "MULTIPLE_OBJECTS"
}, {
"fieldName" : "jsonMaxObjectLen",
"description" : "Larger objects are not processed",
"max" : 2147483647,
"label" : "Max Object Length (chars)",
"required" : true,
"min" : 1,
"dependsOnMap" : {
"dataFormat" : [ "JSON" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 360,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.jsonMaxObjectLen",
"type" : "NUMBER",
"defaultValue" : 4096
}, {
"fieldName" : "csvFileFormat",
"description" : "",
"max" : 9223372036854775807,
"label" : "Delimiter Format Type",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormat" : [ "DELIMITED" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "Default CSV (ignores empty lines)", "RFC4180 CSV", "MS Excel CSV", "MySQL CSV", "Tab Separated Values", "PostgreSQL CSV", "PostgreSQL Text", "Custom", "Multi Character Delimited" ],
"values" : [ "CSV", "RFC4180", "EXCEL", "MYSQL", "TDF", "POSTGRES_CSV", "POSTGRES_TEXT", "CUSTOM", "MULTI_CHARACTER" ],
"configDefinitions" : null,
"modelType" : "VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.pipeline.config.CsvModeChooserValues",
"filteringConfig" : ""
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 370,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.csvFileFormat",
"type" : "MODEL",
"defaultValue" : "CSV"
}, {
"fieldName" : "csvHeader",
"description" : "",
"max" : 9223372036854775807,
"label" : "Header Line",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormat" : [ "DELIMITED" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "With Header Line", "Ignore Header Line", "No Header Line" ],
"values" : [ "WITH_HEADER", "IGNORE_HEADER", "NO_HEADER" ],
"configDefinitions" : null,
"modelType" : "VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.pipeline.config.CsvHeaderChooserValues",
"filteringConfig" : ""
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 380,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.csvHeader",
"type" : "MODEL",
"defaultValue" : "NO_HEADER"
}, {
"fieldName" : "csvAllowExtraColumns",
"description" : "When false, rows with more columns than the header are sent to error.",
"max" : 9223372036854775807,
"label" : "Allow Extra Columns",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormatConfig.csvHeader" : [ "WITH_HEADER" ],
"dataFormat" : [ "DELIMITED" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 385,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.csvAllowExtraColumns",
"type" : "BOOLEAN",
"defaultValue" : false
}, {
"fieldName" : "csvExtraColumnPrefix",
"description" : "Each extra column is labeled with this prefix followed by an integer",
"max" : 9223372036854775807,
"label" : "Extra Column Prefix",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormatConfig.csvHeader" : [ "WITH_HEADER" ],
"dataFormat" : [ "DELIMITED" ],
"dataFormatConfig.csvAllowExtraColumns" : [ true ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 386,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.csvExtraColumnPrefix",
"type" : "STRING",
"defaultValue" : "_extra_"
}, {
"fieldName" : "csvMaxObjectLen",
"description" : "Larger objects are not processed",
"max" : 2147483647,
"label" : "Max Record Length (chars)",
"required" : true,
"min" : 1,
"dependsOnMap" : {
"dataFormat" : [ "DELIMITED" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 390,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.csvMaxObjectLen",
"type" : "NUMBER",
"defaultValue" : 1024
}, {
"fieldName" : "csvCustomDelimiter",
"description" : "",
"max" : 9223372036854775807,
"label" : "Delimiter Character",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormat" : [ "DELIMITED" ],
"dataFormatConfig.csvFileFormat" : [ "CUSTOM" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 400,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.csvCustomDelimiter",
"type" : "CHARACTER",
"defaultValue" : "|"
}, {
"fieldName" : "multiCharacterFieldDelimiter",
"description" : "Delimiter between fields in multi-character delimited mode.",
"max" : 9223372036854775807,
"label" : "Multi Character Field Delimiter",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormat" : [ "DELIMITED" ],
"dataFormatConfig.csvFileFormat" : [ "MULTI_CHARACTER" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 405,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.multiCharacterFieldDelimiter",
"type" : "STRING",
"defaultValue" : "||"
}, {
"fieldName" : "multiCharacterLineDelimiter",
"description" : "Delimiter between lines (i.e. different records) in multi-character delimited mode.",
"max" : 9223372036854775807,
"label" : "Multi Character Line Delimiter",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormat" : [ "DELIMITED" ],
"dataFormatConfig.csvFileFormat" : [ "MULTI_CHARACTER" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 406,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.multiCharacterLineDelimiter",
"type" : "STRING",
"defaultValue" : "${str:unescapeJava('\\\\n')}"
}, {
"fieldName" : "csvCustomEscape",
"description" : "Character used to escape quote and delimiter characters. To disable select Other and enter \\u0000 (unicode codepoint for the NULL character).",
"max" : 9223372036854775807,
"label" : "Escape Character",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormat" : [ "DELIMITED" ],
"dataFormatConfig.csvFileFormat" : [ "MULTI_CHARACTER", "CUSTOM" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 410,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.csvCustomEscape",
"type" : "CHARACTER",
"defaultValue" : "\\"
}, {
"fieldName" : "csvCustomQuote",
"description" : "Character used to quote string fields. To disable select Other and enter \\u0000 (unicode codepoint for the NULL character).",
"max" : 9223372036854775807,
"label" : "Quote Character",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormat" : [ "DELIMITED" ],
"dataFormatConfig.csvFileFormat" : [ "MULTI_CHARACTER", "CUSTOM" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 420,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.csvCustomQuote",
"type" : "CHARACTER",
"defaultValue" : "\""
}, {
"fieldName" : "csvEnableComments",
"description" : "",
"max" : 9223372036854775807,
"label" : "Enable comments",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormat" : [ "DELIMITED" ],
"dataFormatConfig.csvFileFormat" : [ "CUSTOM" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 425,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.csvEnableComments",
"type" : "BOOLEAN",
"defaultValue" : false
}, {
"fieldName" : "csvCommentMarker",
"description" : "",
"max" : 9223372036854775807,
"label" : "Comment marker",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormatConfig.csvEnableComments" : [ true ],
"dataFormat" : [ "DELIMITED" ],
"dataFormatConfig.csvFileFormat" : [ "CUSTOM" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 426,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.csvCommentMarker",
"type" : "CHARACTER",
"defaultValue" : "#"
}, {
"fieldName" : "csvIgnoreEmptyLines",
"description" : "",
"max" : 9223372036854775807,
"label" : "Ignore empty lines",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormat" : [ "DELIMITED" ],
"dataFormatConfig.csvFileFormat" : [ "CUSTOM" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 427,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.csvIgnoreEmptyLines",
"type" : "BOOLEAN",
"defaultValue" : true
}, {
"fieldName" : "csvRecordType",
"description" : "",
"max" : 9223372036854775807,
"label" : "Root Field Type",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormat" : [ "DELIMITED" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "List", "List-Map" ],
"values" : [ "LIST", "LIST_MAP" ],
"configDefinitions" : null,
"modelType" : "VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.pipeline.config.CsvRecordTypeChooserValues",
"filteringConfig" : ""
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 430,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.csvRecordType",
"type" : "MODEL",
"defaultValue" : "LIST_MAP"
}, {
"fieldName" : "csvSkipStartLines",
"description" : "Number of lines to skip before reading",
"max" : 9223372036854775807,
"label" : "Lines to Skip",
"required" : false,
"min" : 0,
"dependsOnMap" : {
"dataFormat" : [ "DELIMITED" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 435,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.csvSkipStartLines",
"type" : "NUMBER",
"defaultValue" : 0
}, {
"fieldName" : "parseNull",
"description" : "When checked, configured string constant will be converted into NULL field.",
"max" : 9223372036854775807,
"label" : "Parse NULLs",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormat" : [ "DELIMITED" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 436,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.parseNull",
"type" : "BOOLEAN",
"defaultValue" : false
}, {
"fieldName" : "nullConstant",
"description" : "String constant that should be converted to a NULL rather then passed as it is.",
"max" : 9223372036854775807,
"label" : "NULL constant",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormat" : [ "DELIMITED" ],
"dataFormatConfig.parseNull" : [ true ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 437,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.nullConstant",
"type" : "STRING",
"defaultValue" : "\\\\N"
}, {
"fieldName" : "xmlRecordElement",
"description" : "XML element that acts as a record delimiter. This may be an element name, with elements having that local name directly under the root as records. Or, it can be a simplified XPath expression (see docs), with elements matching the XPath expression as records. Leaving it blank will treat the whole XML document as one record.",
"max" : 9223372036854775807,
"label" : "Delimiter Element",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormat" : [ "XML" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 440,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.xmlRecordElement",
"type" : "STRING",
"defaultValue" : null
}, {
"fieldName" : "includeFieldXpathAttributes",
"description" : "Include XPath expressions that indicate the path to the input node that was parsed to create each field. If enabled, each field will receive an attribute named \"xpath\" whose value is the XPath expression. Any namespace prefixes will be mapped to the full URI via new xmlns:* attributes in the record header.",
"max" : 9223372036854775807,
"label" : "Include Field XPaths",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormat" : [ "XML" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 442,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.includeFieldXpathAttributes",
"type" : "BOOLEAN",
"defaultValue" : false
}, {
"fieldName" : "xPathNamespaceContext",
"description" : "Namespace context to use if the delimiter is an XPath expression. This should map namespace prefixes to URIs. Any namespace prefix that is used in the record separator expression must be defined here.",
"max" : 9223372036854775807,
"label" : "Namespaces",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormat" : [ "XML" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 445,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.xPathNamespaceContext",
"type" : "MAP",
"defaultValue" : [ ]
}, {
"fieldName" : "outputFieldAttributes",
"description" : "Generate field attributes in output record for XML namespace declarations, and XML attributes. Without this option, they will continue to be output as individual fields (separate from the element value).",
"max" : 9223372036854775807,
"label" : "Output Field Attributes",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormat" : [ "XML" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 448,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.outputFieldAttributes",
"type" : "BOOLEAN",
"defaultValue" : false
}, {
"fieldName" : "xmlMaxObjectLen",
"description" : "Larger records are not processed",
"max" : 2147483647,
"label" : "Max Record Length (chars)",
"required" : true,
"min" : 1,
"dependsOnMap" : {
"dataFormat" : [ "XML" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 450,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.xmlMaxObjectLen",
"type" : "NUMBER",
"defaultValue" : 4096
}, {
"fieldName" : "logMode",
"description" : "",
"max" : 9223372036854775807,
"label" : "Log Format",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormat" : [ "LOG" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "Common Log Format", "Combined Log Format", "Apache Error Log Format", "Apache Access Log Custom Format", "Regular Expression", "Grok Pattern", "Log4j", "Common Event Format (CEF)", "Log Event Extended Format (LEEF)" ],
"values" : [ "COMMON_LOG_FORMAT", "COMBINED_LOG_FORMAT", "APACHE_ERROR_LOG_FORMAT", "APACHE_CUSTOM_LOG_FORMAT", "REGEX", "GROK", "LOG4J", "CEF", "LEEF" ],
"configDefinitions" : null,
"modelType" : "VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.pipeline.config.LogModeChooserValues",
"filteringConfig" : ""
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 460,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.logMode",
"type" : "MODEL",
"defaultValue" : "COMMON_LOG_FORMAT"
}, {
"fieldName" : "logMaxObjectLen",
"description" : "Longer lines are truncated",
"max" : 2147483647,
"label" : "Max Line Length",
"required" : true,
"min" : 1,
"dependsOnMap" : {
"dataFormat" : [ "LOG" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 470,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.logMaxObjectLen",
"type" : "NUMBER",
"defaultValue" : 1024
}, {
"fieldName" : "retainOriginalLine",
"description" : "Indicates if the original line of log should be retained in the record",
"max" : 9223372036854775807,
"label" : "Retain Original Line",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormat" : [ "LOG" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 480,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.retainOriginalLine",
"type" : "BOOLEAN",
"defaultValue" : false
}, {
"fieldName" : "customLogFormat",
"description" : "",
"max" : 9223372036854775807,
"label" : "Custom Log Format",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormatConfig.logMode" : [ "APACHE_CUSTOM_LOG_FORMAT" ],
"dataFormat" : [ "LOG" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 490,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.customLogFormat",
"type" : "STRING",
"defaultValue" : "%h %l %u %t \"%r\" %>s %b"
}, {
"fieldName" : "regex",
"description" : "The regular expression which is used to parse the log line.",
"max" : 9223372036854775807,
"label" : "Regular Expression",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormatConfig.logMode" : [ "REGEX" ],
"dataFormat" : [ "LOG" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 500,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.regex",
"type" : "STRING",
"defaultValue" : "^(\\S+) (\\S+) (\\S+) \\[([\\w:/]+\\s[+\\-]\\d{4})\\] \"(\\S+) (\\S+) (\\S+)\" (\\d{3}) (\\d+)"
}, {
"fieldName" : "fieldPathsToGroupName",
"description" : "Map groups in the regular expression to field paths",
"max" : 9223372036854775807,
"label" : "Field Path To RegEx Group Mapping",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormatConfig.logMode" : [ "REGEX" ],
"dataFormat" : [ "LOG" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : null,
"values" : null,
"configDefinitions" : [ {
"fieldName" : "fieldPath",
"description" : "",
"max" : 9223372036854775807,
"label" : "Field Path",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : { },
"dependsOn" : "",
"group" : "",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 10,
"displayMode" : "BASIC",
"name" : "fieldPath",
"type" : "STRING",
"defaultValue" : "/"
}, {
"fieldName" : "group",
"description" : "",
"max" : 9223372036854775807,
"label" : "Regular Expression Group",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : { },
"dependsOn" : "",
"group" : "",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 20,
"displayMode" : "BASIC",
"name" : "group",
"type" : "NUMBER",
"defaultValue" : 1
} ],
"modelType" : "LIST_BEAN",
"valuesProviderClass" : null,
"filteringConfig" : null
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 510,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.fieldPathsToGroupName",
"type" : "MODEL",
"defaultValue" : [ ]
}, {
"fieldName" : "grokPatternDefinition",
"description" : "Define your own grok patterns which will be used to parse the logs",
"max" : 9223372036854775807,
"label" : "Grok Pattern Definition",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormatConfig.logMode" : [ "GROK" ],
"dataFormat" : [ "LOG" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 520,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.grokPatternDefinition",
"type" : "TEXT",
"defaultValue" : null
}, {
"fieldName" : "grokPattern",
"description" : "The grok pattern which is used to parse the log line",
"max" : 9223372036854775807,
"label" : "Grok Pattern",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormatConfig.logMode" : [ "GROK" ],
"dataFormat" : [ "LOG" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 530,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.grokPattern",
"type" : "STRING",
"defaultValue" : "%{COMMONAPACHELOG}"
}, {
"fieldName" : "onParseError",
"description" : "",
"max" : 9223372036854775807,
"label" : "On Parse Error",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormatConfig.logMode" : [ "LOG4J" ],
"dataFormat" : [ "LOG" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "ERROR", "IGNORE", "INCLUDE AS STACK TRACE" ],
"values" : [ "ERROR", "IGNORE", "INCLUDE_AS_STACK_TRACE" ],
"configDefinitions" : null,
"modelType" : "VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.pipeline.config.OnParseErrorChooserValues",
"filteringConfig" : ""
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 540,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.onParseError",
"type" : "MODEL",
"defaultValue" : "ERROR"
}, {
"fieldName" : "maxStackTraceLines",
"description" : "Any line that does not match the expected pattern will be treated as a Stack trace as long as it is part of the same message. The stack trace will be trimmed to the specified number of lines.",
"max" : 2147483647,
"label" : "Trim Stack Trace to Length",
"required" : true,
"min" : 0,
"dependsOnMap" : {
"dataFormatConfig.logMode" : [ "LOG4J" ],
"dataFormat" : [ "LOG" ],
"dataFormatConfig.onParseError" : [ "INCLUDE_AS_STACK_TRACE" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 550,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.maxStackTraceLines",
"type" : "NUMBER",
"defaultValue" : 50
}, {
"fieldName" : "enableLog4jCustomLogFormat",
"description" : "",
"max" : 9223372036854775807,
"label" : "Use Custom Log Format",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormatConfig.logMode" : [ "LOG4J" ],
"dataFormat" : [ "LOG" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 560,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.enableLog4jCustomLogFormat",
"type" : "BOOLEAN",
"defaultValue" : false
}, {
"fieldName" : "log4jCustomLogFormat",
"description" : "Specify your own custom log4j format.",
"max" : 9223372036854775807,
"label" : "Custom Log4J Format",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormatConfig.logMode" : [ "LOG4J" ],
"dataFormat" : [ "LOG" ],
"dataFormatConfig.enableLog4jCustomLogFormat" : [ true ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 570,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.log4jCustomLogFormat",
"type" : "STRING",
"defaultValue" : "%r [%t] %-5p %c %x - %m%n"
}, {
"fieldName" : "avroSchemaSource",
"description" : "Where to load the Avro Schema from.",
"max" : 9223372036854775807,
"label" : "Avro Schema Location",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormat" : [ "AVRO" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "Message/Data Includes Schema", "In Pipeline Configuration", "Confluent Schema Registry" ],
"values" : [ "SOURCE", "INLINE", "REGISTRY" ],
"configDefinitions" : null,
"modelType" : "VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.pipeline.config.OriginAvroSchemaSourceChooserValues",
"filteringConfig" : ""
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 400,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.avroSchemaSource",
"type" : "MODEL",
"defaultValue" : null
}, {
"fieldName" : "avroSchema",
"description" : "Overrides the schema included in the data (if any). Optionally use the runtime:loadResource function to use a schema stored in a file.",
"max" : 9223372036854775807,
"label" : "Avro Schema",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormat" : [ "AVRO" ],
"dataFormatConfig.avroSchemaSource" : [ "INLINE" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "application/json",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 410,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.avroSchema",
"type" : "TEXT",
"defaultValue" : null
}, {
"fieldName" : "schemaRegistryUrls",
"description" : "List of Confluent Schema Registry URLs",
"max" : 9223372036854775807,
"label" : "Schema Registry URLs",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormat" : [ "AVRO" ],
"dataFormatConfig.avroSchemaSource" : [ "REGISTRY" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 420,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.schemaRegistryUrls",
"type" : "LIST",
"defaultValue" : null
}, {
"fieldName" : "basicAuth",
"description" : "",
"max" : 9223372036854775807,
"label" : "Basic Auth User Info",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormat" : [ "AVRO" ],
"dataFormatConfig.avroSchemaSource" : [ "REGISTRY" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61", "63", "64", "65", "66" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 421,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.basicAuth",
"type" : "CREDENTIAL",
"defaultValue" : null
}, {
"fieldName" : "schemaLookupMode",
"description" : "Whether to look up the Avro Schema by ID or fetch the latest schema for a Subject.",
"max" : 9223372036854775807,
"label" : "Lookup Schema By",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormat" : [ "AVRO" ],
"dataFormatConfig.avroSchemaSource" : [ "REGISTRY" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "Subject", "Schema ID", "Embedded Schema ID" ],
"values" : [ "SUBJECT", "ID", "AUTO" ],
"configDefinitions" : null,
"modelType" : "VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.pipeline.config.OriginAvroSchemaLookupModeChooserValues",
"filteringConfig" : ""
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 430,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.schemaLookupMode",
"type" : "MODEL",
"defaultValue" : "SUBJECT"
}, {
"fieldName" : "subject",
"description" : "",
"max" : 9223372036854775807,
"label" : "Schema Subject",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormat" : [ "AVRO" ],
"dataFormatConfig.schemaLookupMode" : [ "SUBJECT" ],
"dataFormatConfig.avroSchemaSource" : [ "REGISTRY" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 440,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.subject",
"type" : "STRING",
"defaultValue" : null
}, {
"fieldName" : "schemaId",
"description" : "",
"max" : 9223372036854775807,
"label" : "Schema ID",
"required" : true,
"min" : 1,
"dependsOnMap" : {
"dataFormat" : [ "AVRO" ],
"dataFormatConfig.schemaLookupMode" : [ "ID" ],
"dataFormatConfig.avroSchemaSource" : [ "REGISTRY" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 450,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.schemaId",
"type" : "NUMBER",
"defaultValue" : null
}, {
"fieldName" : "avroSkipUnionIndex",
"description" : "When checked generated records will not contain header attributes identifying which part of a union was used to read data in. Data Collector does not use the header attributes directly, thus this can be selected safely unless the pipeline explicitly depends on them.",
"max" : 9223372036854775807,
"label" : "Skip Union Indexes",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormat" : [ "AVRO" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 460,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.avroSkipUnionIndex",
"type" : "BOOLEAN",
"defaultValue" : false
}, {
"fieldName" : "protoDescriptorFile",
"description" : "Protobuf Descriptor File (.desc) path relative to SDC resources directory",
"max" : 9223372036854775807,
"label" : "Protobuf Descriptor File",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormat" : [ "PROTOBUF" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 600,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.protoDescriptorFile",
"type" : "STRING",
"defaultValue" : null
}, {
"fieldName" : "messageType",
"description" : "Fully Qualified Message Type name. Use format <packageName>.<messageTypeName>",
"max" : 9223372036854775807,
"label" : "Message Type",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormat" : [ "PROTOBUF" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 610,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.messageType",
"type" : "STRING",
"defaultValue" : null
}, {
"fieldName" : "isDelimited",
"description" : "Should be checked when the input data is prepended with the message size. When unchecked only a single message can be present in the source file/Kafka message, etc.",
"max" : 9223372036854775807,
"label" : "Delimited Messages",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormat" : [ "PROTOBUF" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 620,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.isDelimited",
"type" : "BOOLEAN",
"defaultValue" : true
}, {
"fieldName" : "binaryMaxObjectLen",
"description" : "Larger objects are not processed",
"max" : 2147483647,
"label" : "Max Data Size (bytes)",
"required" : true,
"min" : 1,
"dependsOnMap" : {
"dataFormat" : [ "BINARY" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 700,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.binaryMaxObjectLen",
"type" : "NUMBER",
"defaultValue" : 1024
}, {
"fieldName" : "datagramMode",
"description" : "",
"max" : 9223372036854775807,
"label" : "Datagram Packet Format",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormat" : [ "DATAGRAM" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "NetFlow", "syslog", "collectd", "Raw/Separated data" ],
"values" : [ "NETFLOW", "SYSLOG", "COLLECTD", "RAW_DATA" ],
"configDefinitions" : null,
"modelType" : "VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.pipeline.config.DatagramModeChooserValues",
"filteringConfig" : ""
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 800,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.datagramMode",
"type" : "MODEL",
"defaultValue" : "SYSLOG"
}, {
"fieldName" : "typesDbPath",
"description" : "User-specified TypesDB file. Overrides the included version.",
"max" : 9223372036854775807,
"label" : "TypesDB File Path",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormat" : [ "DATAGRAM" ],
"dataFormatConfig.datagramMode" : [ "COLLECTD" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 820,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.typesDbPath",
"type" : "STRING",
"defaultValue" : null
}, {
"fieldName" : "convertTime",
"description" : "Converts high resolution time format interval and timestamp to unix time in (ms).",
"max" : 9223372036854775807,
"label" : "Convert Hi-Res Time & Interval",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormat" : [ "DATAGRAM" ],
"dataFormatConfig.datagramMode" : [ "COLLECTD" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 830,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.convertTime",
"type" : "BOOLEAN",
"defaultValue" : false
}, {
"fieldName" : "excludeInterval",
"description" : "Excludes the interval field from output records.",
"max" : 9223372036854775807,
"label" : "Exclude Interval",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormat" : [ "DATAGRAM" ],
"dataFormatConfig.datagramMode" : [ "COLLECTD" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 840,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.excludeInterval",
"type" : "BOOLEAN",
"defaultValue" : true
}, {
"fieldName" : "authFilePath",
"description" : "",
"max" : 9223372036854775807,
"label" : "Auth File",
"required" : false,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormat" : [ "DATAGRAM" ],
"dataFormatConfig.datagramMode" : [ "COLLECTD" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 850,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.authFilePath",
"type" : "STRING",
"defaultValue" : null
}, {
"fieldName" : "netflowOutputValuesMode",
"description" : "Determines the data that is included in the record generated from a flow. Only applies to NetFlow 9.",
"max" : 9223372036854775807,
"label" : "Record Generation Mode",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormat" : [ "NETFLOW" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "Raw only", "Interpreted only", "Raw and interpreted" ],
"values" : [ "RAW_ONLY", "INTERPRETED_ONLY", "RAW_AND_INTERPRETED" ],
"configDefinitions" : null,
"modelType" : "VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.pipeline.lib.parser.net.netflow.OutputValuesModeChooserValues",
"filteringConfig" : ""
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 870,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.netflowOutputValuesMode",
"type" : "MODEL",
"defaultValue" : "RAW_AND_INTERPRETED"
}, {
"fieldName" : "maxTemplateCacheSize",
"description" : "Controls the maximum number of templates to cache from all sources. Additional templates received when this limit is reached cause the eviction of existing templates, least recently used first. Leave as -1 for unlimited. Only applies to NetFlow 9.",
"max" : 9223372036854775807,
"label" : "Max Templates in Cache",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormat" : [ "NETFLOW" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 880,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.maxTemplateCacheSize",
"type" : "NUMBER",
"defaultValue" : -1
}, {
"fieldName" : "templateCacheTimeoutMs",
"description" : "Controls the maximum length of time flow templates are cached, after last being used to parse a data flow. Leave as -1 for unlimited (never expires). Only applies to NetFlow 9.",
"max" : 9223372036854775807,
"label" : "Template Cache Timeout (ms)",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormat" : [ "NETFLOW" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 890,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.templateCacheTimeoutMs",
"type" : "NUMBER",
"defaultValue" : -1
}, {
"fieldName" : "netflowOutputValuesModeDatagram",
"description" : "Determines the data that is included in the record generated from a flow. Only applies to NetFlow 9.",
"max" : 9223372036854775807,
"label" : "Record Generation Mode",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormat" : [ "DATAGRAM" ],
"dataFormatConfig.datagramMode" : [ "NETFLOW" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "Raw only", "Interpreted only", "Raw and interpreted" ],
"values" : [ "RAW_ONLY", "INTERPRETED_ONLY", "RAW_AND_INTERPRETED" ],
"configDefinitions" : null,
"modelType" : "VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.pipeline.lib.parser.net.netflow.OutputValuesModeChooserValues",
"filteringConfig" : ""
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 870,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.netflowOutputValuesModeDatagram",
"type" : "MODEL",
"defaultValue" : "RAW_AND_INTERPRETED"
}, {
"fieldName" : "maxTemplateCacheSizeDatagram",
"description" : "Controls the maximum number of templates to cache from all sources. Additional templates received when this limit is reached cause the eviction of existing templates, least recently used first. Leave as -1 for unlimited. Only applies to NetFlow 9.",
"max" : 9223372036854775807,
"label" : "Max Templates in Cache",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormat" : [ "DATAGRAM" ],
"dataFormatConfig.datagramMode" : [ "NETFLOW" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 880,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.maxTemplateCacheSizeDatagram",
"type" : "NUMBER",
"defaultValue" : -1
}, {
"fieldName" : "templateCacheTimeoutMsDatagram",
"description" : "Controls the maximum length of time flow templates are cached, after last being used to parse a data flow. Leave as -1 for unlimited (never expires). Only applies to NetFlow 9.",
"max" : 9223372036854775807,
"label" : "Template Cache Timeout (ms)",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormat" : [ "DATAGRAM" ],
"dataFormatConfig.datagramMode" : [ "NETFLOW" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 890,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.templateCacheTimeoutMsDatagram",
"type" : "NUMBER",
"defaultValue" : -1
}, {
"fieldName" : "wholeFileMaxObjectLen",
"description" : "Size of the Buffer used to copy the file.",
"max" : 2147483647,
"label" : "Buffer Size (bytes)",
"required" : true,
"min" : 1,
"dependsOnMap" : {
"dataFormat" : [ "WHOLE_FILE" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 900,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.wholeFileMaxObjectLen",
"type" : "NUMBER",
"defaultValue" : 8192
}, {
"fieldName" : "rateLimit",
"description" : "Rate / sec to manipulate bandwidth requirements for File Transfer. Use <= 0 to opt out. Default unit is B/sec",
"max" : 9223372036854775807,
"label" : "Rate per second",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormat" : [ "WHOLE_FILE" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "EXPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62", "267", "268", "269" ],
"displayPosition" : 920,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.rateLimit",
"type" : "STRING",
"defaultValue" : "-1"
}, {
"fieldName" : "verifyChecksum",
"description" : "When checked verifies the checksum of the stream during read.",
"max" : 9223372036854775807,
"label" : "Verify Checksum",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormat" : [ "WHOLE_FILE" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 1000,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.verifyChecksum",
"type" : "BOOLEAN",
"defaultValue" : false
}, {
"fieldName" : "excelHeader",
"description" : "Excel headers",
"max" : 9223372036854775807,
"label" : "Excel Header Option",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormat" : [ "EXCEL" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : {
"labels" : [ "With Header Line", "Ignore Header Line", "No Header Line" ],
"values" : [ "WITH_HEADER", "IGNORE_HEADER", "NO_HEADER" ],
"configDefinitions" : null,
"modelType" : "VALUE_CHOOSER",
"valuesProviderClass" : "com.streamsets.pipeline.config.ExcelHeaderChooserValues",
"filteringConfig" : ""
},
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 1000,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.excelHeader",
"type" : "MODEL",
"defaultValue" : null
}, {
"fieldName" : "excelSkipCellsWithNoHeader",
"description" : "If checked, cells that have no associated header value will be skipped.",
"max" : 9223372036854775807,
"label" : "Skip Cells With No Header",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormat" : [ "EXCEL" ],
"dataFormatConfig.excelHeader" : [ "WITH_HEADER" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 1001,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.excelSkipCellsWithNoHeader",
"type" : "BOOLEAN",
"defaultValue" : false
}, {
"fieldName" : "excelReadAllSheets",
"description" : "Specifies whether all sheets from the document should be read.",
"max" : 9223372036854775807,
"label" : "Read All Sheets",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormat" : [ "EXCEL" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ ],
"elConstantDefinitionsIdx" : [ ],
"displayPosition" : 1010,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.excelReadAllSheets",
"type" : "BOOLEAN",
"defaultValue" : true
}, {
"fieldName" : "excelSheetNames",
"description" : "Names of the sheets that should be imported. Other sheets will be ignored.",
"max" : 9223372036854775807,
"label" : "Import Sheets",
"required" : true,
"min" : -9223372036854775808,
"dependsOnMap" : {
"dataFormatConfig.excelReadAllSheets" : [ false ],
"dataFormat" : [ "EXCEL" ]
},
"dependsOn" : "",
"group" : "DATA_FORMAT",
"evaluation" : "IMPLICIT",
"elDefs" : null,
"model" : null,
"mode" : "text/plain",
"lines" : 0,
"triggeredByValues" : null,
"elFunctionDefinitionsIdx" : [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61" ],
"elConstantDefinitionsIdx" : [ "62" ],
"displayPosition" : 1020,
"displayMode" : "BASIC",
"name" : "dataFormatConfig.excelSheetNames",
"type" : "LIST",
"defaultValue" : [ ]
} ],
"configGroupDefinition" : {
"classNameToGroupsMap" : {
"com.streamsets.service.parser.Groups" : [ "DATA_FORMAT" ]
},
"groupNameToLabelMapList" : [ {
"name" : "DATA_FORMAT",
"label" : "Data Format"
} ]
},
"libraryLabel" : "Data Formats",
"className" : "com.streamsets.service.parser.DataParserServiceImpl"
} ],
"rulesElMetadata" : null,
"elCatalog" : null,
"runtimeConfigs" : null,
"stageIcons" : {
"streamsets-datacollector-dev-lib:com_streamsets_pipeline_stage_devtest_rawdata_RawDataDSource" : "iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAYAAABXAvmHAAADXUlEQVRoQ+2XWchOQRjHf5+lEMqaCxf2ZI/IVoQLkaWI0BclbhBKKLJcKJTIVkJSiCRKFEUiyS4KWSNcIMp2Ye+v52ga55xvcpz01czVe87MPGf+y/M881ZQzUdFNT8/EcD/VjAqEBUoyEC0UEECC2+PChSmsGCAqEBBAgtvjwoUprBggCwFWgBdga9e/FrAc+AO8MOb6wU0DDjPZ+AC0BjoAXyzPTWBl8BNe+4GNHHi6dtXgTfuN7IATAN25RzmMjAJeGhrFOcwMCYAgA7Z0db637gN6OB1gEdAcy/eAmBdCIA+wDJgJHANOGKbBKyNE6AzoI8KwGygOzDd5g8CT4Eazvr5xrIAtAI2AQNsfjdwHthhz7OAtsA8exZpin0rBIDW6AOPgRlOUL0fCBwyds4BQxyr1QauALJga+CTx+BGYKIp8BZoCrwyULKs1HGHGy9tPvcPTTvgPiAmtnqBxcwDe9ffPK3HevZbAMSyDtke6AvsA9Z7AKScbDQVkLpSwR2KLVW0txL47ls0r4zmAVCclWYzWWeLBZZ3L5oCOvg7y5UNQAdgMjAKmAB8sD0jgGOAr6amVwOLgNHA0bT8KgJgrCVuGgAl4mlL8qFAfUcR/xyNgLtmSYGU6hqqaNdtbxez2h8YigAYDyhRxZ6qj8phooAAuCOpPLJU2lBuzPHsOgg4A+y0PPTL9q84/wLAWmCx9QXXQvL9C/PuqhwFdI7ksPcAsf0FSEANB05kAC8EQH7e67HmAkiSeBiwzQ6mdy2B404DS+yi5tYJUGl+Yg0tz3qFFFBtP2s1XA1tf0oSJwBc8g4AgzPUWA6sMCvdsKRWn5ib0vV/xwyx0ExguyehGooaju/ttCrkbvX7gDvXG7gEvLdE7gmoAKgYZI4sALqXqAaL5SVWv9VU1IWXAuMsotsjGlglOQk0s9r/2skzJblA98tQwC8Az+xakZX4uRaaAuzJQw5stjafVB9dtOThqkZeRVoIrLEAUkv2yR1ZCiTNxd8seeV3efmUM6mbou5LujtVNXRJ0y1UsfyRXF/0XpbSteSvAFS1r8z5unYB/BjykfiPLISlMtdEBcpkNyR2VCCEpTLXRAXKZDckdlQghKUy10QFymQ3JHZUIISlMtdEBcpkNyR2tVfgJ7j+uDHAB/IgAAAAAElFTkSuQmCC",
"streamsets-datacollector-basic-lib:com_streamsets_pipeline_stage_processor_fieldtypeconverter_FieldTypeConverterDProcessor" : "iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAYAAABXAvmHAAABdUlEQVRoQ+2ZAQ0CMQxFDwVIAByAAnAAEg4JOMEB4ACcIAEJOIA2Ycndsruu6y7dSJcsubBd+1+79g6YNZWPWeX6m1SAFsC3MI/aAUgBQPGXn/CrNgQXoCveBV8VggMQEq8OEQswJl4VIgYgRrwaRAyA32g+3gcpNrI1rxTnBpAt/GDoLzPgHxEqYH4QuPe/wMET5gkmXrNGKANcAVIAJ/gNFxsuREkACPKAeeCkoDQAPEIrKQB1f+42KrJXQhcyAOrITP0qYRmwDHAj4O23I8QNoChiAWcie/YcSHwl7ybCMqBdA1z/vf0l1IABcCMgKjquM2p/yhHCr35zyvCE6z3NKQB3ELefUCBlWgywBA/4K4JWFsQAGCGEOMNcw1xQIcu8ngUgpyZRU0ipgZzi0ZYB5I4o1151GWiB0P1JSMHeYAPuHxxaNRADQYpHKi0A9D0GESVeG2AIIlp8CQA+BEt8KQAOYkcVbKiSNWuA6kBR69UDfAGOsk0xTaeqMAAAAABJRU5ErkJggg==",
"streamsets-datacollector-basic-lib:com_streamsets_pipeline_stage_destination_devnull_NullDTarget" : "iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAYAAABXAvmHAAAABmJLR0QA/wD/AP+gvaeTAAACSUlEQVRoge3YPWsVQRQG4CcaMZpLCJKksLC2sxAstLGz9weIIthaiJUWASuxtdbGwv9grZ02NlYBCQiSaAgkBL8Si9nFzd6P7M6Me0X2heHCcM6ZOfO+Z+fcoUePHj3+FyzgOTZxMGZs4FlhmwWzuQLhKa7hET6NsTmLB5jBrYxrZ8E27jawu4etXIseyxVIkEWTjX3FYq5F20joYjEm4SrmjrC5XPzemWDzthhZsWp8ceYeq7k3X8VDfPwLcd/jSVunmBrYwEqE31FYKWK3QmwCcxhE+E7axxnhDmnt2BblKS3X5r/jZm3uZTGquI1vtblF4YPSGQMMJ7CrGSuDwraKMtZUGdjRPIGd2lwZqxMGtvDLaAbmG/jPG2ZgqfjthIF94TYdxUDTBEYx8ENoR1ohtpXYkL8Gyi62FXInkCKh1vIhbwJNi3ichFoXMNNhYJSElvwDCaQy0KmENuVnoPMaGOBUZa7pV2hUEU+lBvhzARH/FTpdjM4Z4LCMYluJ6DaCvAnsCm328Ql+szjpMAPRbQRpRXxgmIEZQQ7jMF+xLTEVBsq+pc4Ak2VUJlBlYFk4jE4ZYPhTWm5qUiGXydUltC0cSmukJFC/zEpZNGGgLqGo0ydvAikMROmfnoHWDIwq4iQGUl6n60W8h8dYq8y9qvmsFTZ7lbllvEnYRzRuCH8vTyTG+YL7sc4pEloXLq5zCTEWhDeh9dgAMwmLD/BBeCd9IbxUtF37Oi7hPD4n7CUaF/AaP7V/gd7HO1zpfNc9evTokQ2/AbbOuP9A2WXaAAAAAElFTkSuQmCC",
"streamsets-datacollector-dev-lib:com_streamsets_pipeline_stage_devtest_RandomDataGeneratorSource" : "iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAYAAABXAvmHAAADXUlEQVRoQ+2XWchOQRjHf5+lEMqaCxf2ZI/IVoQLkaWI0BclbhBKKLJcKJTIVkJSiCRKFEUiyS4KWSNcIMp2Ye+v52ga55xvcpz01czVe87MPGf+y/M881ZQzUdFNT8/EcD/VjAqEBUoyEC0UEECC2+PChSmsGCAqEBBAgtvjwoUprBggCwFWgBdga9e/FrAc+AO8MOb6wU0DDjPZ+AC0BjoAXyzPTWBl8BNe+4GNHHi6dtXgTfuN7IATAN25RzmMjAJeGhrFOcwMCYAgA7Z0db637gN6OB1gEdAcy/eAmBdCIA+wDJgJHANOGKbBKyNE6AzoI8KwGygOzDd5g8CT4Eazvr5xrIAtAI2AQNsfjdwHthhz7OAtsA8exZpin0rBIDW6AOPgRlOUL0fCBwyds4BQxyr1QauALJga+CTx+BGYKIp8BZoCrwyULKs1HGHGy9tPvcPTTvgPiAmtnqBxcwDe9ffPK3HevZbAMSyDtke6AvsA9Z7AKScbDQVkLpSwR2KLVW0txL47ls0r4zmAVCclWYzWWeLBZZ3L5oCOvg7y5UNQAdgMjAKmAB8sD0jgGOAr6amVwOLgNHA0bT8KgJgrCVuGgAl4mlL8qFAfUcR/xyNgLtmSYGU6hqqaNdtbxez2h8YigAYDyhRxZ6qj8phooAAuCOpPLJU2lBuzPHsOgg4A+y0PPTL9q84/wLAWmCx9QXXQvL9C/PuqhwFdI7ksPcAsf0FSEANB05kAC8EQH7e67HmAkiSeBiwzQ6mdy2B404DS+yi5tYJUGl+Yg0tz3qFFFBtP2s1XA1tf0oSJwBc8g4AgzPUWA6sMCvdsKRWn5ib0vV/xwyx0ExguyehGooaju/ttCrkbvX7gDvXG7gEvLdE7gmoAKgYZI4sALqXqAaL5SVWv9VU1IWXAuMsotsjGlglOQk0s9r/2skzJblA98tQwC8Az+xakZX4uRaaAuzJQw5stjafVB9dtOThqkZeRVoIrLEAUkv2yR1ZCiTNxd8seeV3efmUM6mbou5LujtVNXRJ0y1UsfyRXF/0XpbSteSvAFS1r8z5unYB/BjykfiPLISlMtdEBcpkNyR2VCCEpTLXRAXKZDckdlQghKUy10QFymQ3JHZUIISlMtdEBcpkNyR2tVfgJ7j+uDHAB/IgAAAAAElFTkSuQmCC",
"streamsets-datacollector-basic-lib:com_streamsets_pipeline_stage_destination_devnull_StatsDpmDirectlyDTarget" : null,
"streamsets-datacollector-basic-lib:com_streamsets_pipeline_stage_processor_expression_ExpressionDProcessor" : "iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAYAAABXAvmHAAAABmJLR0QA/wD/AP+gvaeTAAAD90lEQVRoge3ZXYhVVRQH8J+TZkyW9IE1FGpiWRj0gVIgmQkmFD0kvoRQImUfL1EPRa/VUy+BD5lhPQRFmlSWfakEQRARZIph0Qc4hWLlmFbSqDO3h3Vvc+65+5x7ZubO9HL/cF72WWvt/9p7nbXXXocuuuiiiy7GgSmTONdC3IGezNgADmaefyaRz5gwB/dgMw6jlnsOYCPuxPT/iWNlTMESbMFprc4cwwYs6PTEZ2ERHsGlHbJ5Bd7V6kRNOPcCLhrPBPPxELbhKPabgJXBapyQduQXLKtqaC7WYyt+zRn6GOd2jnMLrhdkU06cqfMqxFp8jeECAx+hd2J4N2Eh/izgMIwHihR7hIfHE4q7cPYoicwQ274eT4gFWq5ahlld4EANg7ixTHmeiPOGwo9G9xEtFeH3t8j1+8T2N+wNiG+qHXaUOPFVmeJynDISdzdXJH453sxM8pKR7+XaOvEsifvb2FsgnWIbz+0ppbk4khHaWJH8rTm9txIym3MEDlSwu7vEgU154V7szQgMilVth5VGdqyGQ9IhtylH4FgF2w+XOLAvL/xiTuCVChMs1PrRP1Yguz8nt6uC/b4SBwaygmsSAskYy6AHX+R0jkufE0tycmfqY1XwW4EDfzUEZmn9wAYLiGRxX8Lo1oRcr9juLPm1FcnDZwUOfNsQ2JB4uaeC4c8TenliM/Fh5v0PuGUU5OH1Age2EQXZYOLlzjZGZ0uf2NfU358nDrD++vhBcYJOGyV5eKPAgXunYpX0Cft7G6M3ab0QDeNBLMYNdfI7sV3UUKfGQB7OSYydxDtTxeUhhXY1z5zE2JCoYZ4SNdWJqgzbYFZibEPD/k/S2/NlG6OPJ3Te7wzfJkwXV83sPD+LENWDywoU2x1g/Ymx4bFxLMUizcVfDevETiNOw6KD4soSw32aC7Sa2M0qWIbrKsq+nJvjubzAd4odKKy769iS0FlRIt+H9/CNdFyn5LMZ8gPNXQ20eph9tleYIN9dOCyyUBbTRfk8gE8rkqe5tOkv0ru7xIEhI3m9CFfj+4TeJ6J4e7tO/CgexdSK5JcaOWf+EFfNJHrElhY58WqFyaaJWmqLCMkBcY7swWui7p9RkThcIi5RDQ67tblQrRSrlnJgWHTUJgsXaq1aayLZrFfSTXwmodR4jihOt53EBYoLt5rY2dLE8qTijsRB5Wl1vJivOWwahLeKlZ9X1dA6xc2lflEDdRorxC3utKhwn8VtxtEfnS1SaMqJ03h6PMZzWInncRfO75DN/7BYpMFUZ+CQ6PXM7PSkE4E+Ubzt0BpeJ0UrZZUJWMF2GMsPjikixK7CxepVYR1DouG018QUdl100UUXXXQW/wKLgdYjXL+F5AAAAABJRU5ErkJggg==",
"streamsets-datacollector-basic-lib:com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget" : null
},
"legacyStageLibs" : null,
"eventDefinitions" : null,
"version" : null,
"executorVersion" : null,
"category" : null,
"categoryLabel" : null
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment