Skip to content

Instantly share code, notes, and snippets.

@metadaddy

metadaddy/S3 to Azure Synapse.json

Last active November 7, 2019 21:54
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save metadaddy/79390152dbcfad4a43dbbc888fce6127 to your computer and use it in GitHub Desktop.
Save metadaddy/79390152dbcfad4a43dbbc888fce6127 to your computer and use it in GitHub Desktop.
Download ZIP
Use StreamSets Cloud to read data from Amazon S3 and write to Azure Synapse (formerly SQL DW)
Raw
S3 to Azure Synapse.json
{
"ciConfig" : null,
"currentExternalCIStatus" : null,
"currentRules" : {
"commitId" : "55b9a63d-8712-4a8d-96bf-511618889159:dpmfrancois",
"definitionRemoveMessage" : null,
"definitionRemoveTime" : 0,
"definitionRemover" : null,
"id" : "d53e9e53-eb4c-4204-9227-fd40d3a1b440:dpmfrancois",
"message" : "Committed with pipeline",
"name" : "Initial rules",
"rulesCommitter" : "admin@dpmfrancois",
"rulesDefinition" : "{\"schemaVersion\":3,\"version\":2,\"metricsRuleDefinitions\":[{\"id\":\"badRecordsAlertID\",\"alertText\":\"High incidence of Error Records\",\"metricId\":\"pipeline.batchErrorRecords.counter\",\"metricType\":\"COUNTER\",\"metricElement\":\"COUNTER_COUNT\",\"condition\":\"${value() > 100}\",\"sendEmail\":false,\"enabled\":false,\"timestamp\":1554928504911,\"valid\":true},{\"id\":\"stageErrorAlertID\",\"alertText\":\"High incidence of Stage Errors\",\"metricId\":\"pipeline.batchErrorMessages.counter\",\"metricType\":\"COUNTER\",\"metricElement\":\"COUNTER_COUNT\",\"condition\":\"${value() > 100}\",\"sendEmail\":false,\"enabled\":false,\"timestamp\":1554928504911,\"valid\":true},{\"id\":\"idleGaugeID\",\"alertText\":\"Pipeline is Idle\",\"metricId\":\"RuntimeStatsGauge.gauge\",\"metricType\":\"GAUGE\",\"metricElement\":\"TIME_OF_LAST_RECEIVED_RECORD\",\"condition\":\"${time:now() - value() > 120000}\",\"sendEmail\":false,\"enabled\":false,\"timestamp\":1554928504911,\"valid\":true},{\"id\":\"batchTimeAlertID\",\"alertText\":\"Batch taking more time to process\",\"metricId\":\"RuntimeStatsGauge.gauge\",\"metricType\":\"GAUGE\",\"metricElement\":\"CURRENT_BATCH_AGE\",\"condition\":\"${value() > 200}\",\"sendEmail\":false,\"enabled\":false,\"timestamp\":1554928504911,\"valid\":true}],\"dataRuleDefinitions\":[],\"driftRuleDefinitions\":[],\"uuid\":\"3ce84177-6103-4f66-a4ee-91b9cf157662\",\"configuration\":[{\"name\":\"emailIDs\",\"value\":[]},{\"name\":\"webhookConfigs\",\"value\":[]}],\"ruleIssues\":[],\"configIssues\":[]}",
"rulesTime" : 1554928504791
},
"definitionRemoveMessage" : null,
"definitionRemoveTime" : 0,
"definitionRemover" : null,
"executionMode" : "STANDALONE",
"externalCiStatuses" : [ ],
"fragment" : false,
"fragmentCommitIds" : [ ],
"libraryDefinitions" : null,
"pipelineDefinition" : "{\n \"schemaVersion\" : 6,\n \"version\" : 15,\n \"pipelineId\" : \"retertfe48397d-0ee4-450c-a7f2-6a40ba666a73\",\n \"title\" : \"S3 to Azure Synapse\",\n \"description\" : \"\",\n \"uuid\" : \"c43e05e5-b890-4bab-a8a3-69432b244641\",\n \"configuration\" : [ {\n \"name\" : \"executionMode\",\n \"value\" : \"STANDALONE\"\n }, {\n \"name\" : \"edgeHttpUrl\",\n \"value\" : \"http://localhost:18633\"\n }, {\n \"name\" : \"deliveryGuarantee\",\n \"value\" : \"AT_LEAST_ONCE\"\n }, {\n \"name\" : \"testOriginStage\",\n \"value\" : null\n }, {\n \"name\" : \"startEventStage\",\n \"value\" : null\n }, {\n \"name\" : \"stopEventStage\",\n \"value\" : null\n }, {\n \"name\" : \"shouldRetry\",\n \"value\" : true\n }, {\n \"name\" : \"retryAttempts\",\n \"value\" : -1\n }, {\n \"name\" : \"notifyOnStates\",\n \"value\" : [ \"RUN_ERROR\", \"STOPPED\", \"FINISHED\" ]\n }, {\n \"name\" : \"emailIDs\",\n \"value\" : [ ]\n }, {\n \"name\" : \"constants\",\n \"value\" : [ ]\n }, {\n \"name\" : \"badRecordsHandling\",\n \"value\" : \"streamsets-cloud-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1\"\n }, {\n \"name\" : \"errorRecordPolicy\",\n \"value\" : \"ORIGINAL_RECORD\"\n }, {\n \"name\" : \"statsAggregatorStage\",\n \"value\" : null\n }, {\n \"name\" : \"workerCount\",\n \"value\" : 0\n }, {\n \"name\" : \"clusterSlaveMemory\",\n \"value\" : 2048\n }, {\n \"name\" : \"clusterSlaveJavaOpts\",\n \"value\" : \"-XX:+UseConcMarkSweepGC -XX:+UseParNewGC -Dlog4j.debug\"\n }, {\n \"name\" : \"clusterLauncherEnv\",\n \"value\" : [ ]\n }, {\n \"name\" : \"mesosDispatcherURL\",\n \"value\" : null\n }, {\n \"name\" : \"logLevel\",\n \"value\" : \"INFO\"\n }, {\n \"name\" : \"hdfsS3ConfDir\",\n \"value\" : null\n }, {\n \"name\" : \"rateLimit\",\n \"value\" : 0\n }, {\n \"name\" : \"maxRunners\",\n \"value\" : 0\n }, {\n \"name\" : \"shouldCreateFailureSnapshot\",\n \"value\" : true\n }, {\n \"name\" : \"runnerIdleTIme\",\n \"value\" : 60\n }, {\n \"name\" : \"webhookConfigs\",\n \"value\" : [ ]\n }, {\n \"name\" : \"sparkConfigs\",\n \"value\" : [ ]\n }, {\n \"name\" : \"amazonEMRConfig.userRegion\",\n \"value\" : null\n }, {\n \"name\" : \"amazonEMRConfig.userRegionCustom\",\n \"value\" : null\n }, {\n \"name\" : \"amazonEMRConfig.accessKey\",\n \"value\" : null\n }, {\n \"name\" : \"amazonEMRConfig.secretKey\",\n \"value\" : null\n }, {\n \"name\" : \"amazonEMRConfig.s3StagingUri\",\n \"value\" : null\n }, {\n \"name\" : \"amazonEMRConfig.provisionNewCluster\",\n \"value\" : false\n }, {\n \"name\" : \"amazonEMRConfig.clusterId\",\n \"value\" : null\n }, {\n \"name\" : \"amazonEMRConfig.clusterPrefix\",\n \"value\" : null\n }, {\n \"name\" : \"amazonEMRConfig.terminateCluster\",\n \"value\" : false\n }, {\n \"name\" : \"amazonEMRConfig.loggingEnabled\",\n \"value\" : true\n }, {\n \"name\" : \"amazonEMRConfig.s3LogUri\",\n \"value\" : null\n }, {\n \"name\" : \"amazonEMRConfig.enableEMRDebugging\",\n \"value\" : true\n }, {\n \"name\" : \"amazonEMRConfig.serviceRole\",\n \"value\" : \"EMR_DefaultRole\"\n }, {\n \"name\" : \"amazonEMRConfig.jobFlowRole\",\n \"value\" : \"EMR_EC2_DefaultRole\"\n }, {\n \"name\" : \"amazonEMRConfig.visibleToAllUsers\",\n \"value\" : true\n }, {\n \"name\" : \"amazonEMRConfig.ec2SubnetId\",\n \"value\" : null\n }, {\n \"name\" : \"amazonEMRConfig.masterSecurityGroup\",\n \"value\" : null\n }, {\n \"name\" : \"amazonEMRConfig.slaveSecurityGroup\",\n \"value\" : null\n }, {\n \"name\" : \"amazonEMRConfig.instanceCount\",\n \"value\" : 2\n }, {\n \"name\" : \"amazonEMRConfig.masterInstanceType\",\n \"value\" : null\n }, {\n \"name\" : \"amazonEMRConfig.masterInstanceTypeCustom\",\n \"value\" : null\n }, {\n \"name\" : \"amazonEMRConfig.slaveInstanceType\",\n \"value\" : null\n }, {\n \"name\" : \"amazonEMRConfig.slaveInstanceTypeCustom\",\n \"value\" : null\n }, {\n \"name\" : \"triggerInterval\",\n \"value\" : 2000\n }, {\n \"name\" : \"ludicrousMode\",\n \"value\" : false\n }, {\n \"name\" : \"ludicrousModeInputCount\",\n \"value\" : false\n }, {\n \"name\" : \"advancedErrorHandling\",\n \"value\" : false\n }, {\n \"name\" : \"clusterConfig.clusterType\",\n \"value\" : \"LOCAL\"\n }, {\n \"name\" : \"clusterConfig.sparkMasterUrl\",\n \"value\" : \"local[*]\"\n }, {\n \"name\" : \"clusterConfig.deployMode\",\n \"value\" : \"CLIENT\"\n }, {\n \"name\" : \"clusterConfig.hadoopUserName\",\n \"value\" : null\n }, {\n \"name\" : \"clusterConfig.sparkAppName\",\n \"value\" : \"${pipeline:title()}\"\n }, {\n \"name\" : \"clusterConfig.stagingDir\",\n \"value\" : \"/streamsets\"\n }, {\n \"name\" : \"clusterConfig.useYarnKerberosKeytab\",\n \"value\" : false\n }, {\n \"name\" : \"clusterConfig.yarnKerberosKeytabSource\",\n \"value\" : \"PROPERTIES_FILE\"\n }, {\n \"name\" : \"clusterConfig.yarnKerberosKeytab\",\n \"value\" : null\n }, {\n \"name\" : \"clusterConfig.yarnKerberosPrincipal\",\n \"value\" : \"name@DOMAIN\"\n }, {\n \"name\" : \"databricksConfig.baseUrl\",\n \"value\" : null\n }, {\n \"name\" : \"databricksConfig.credentialType\",\n \"value\" : null\n }, {\n \"name\" : \"databricksConfig.username\",\n \"value\" : null\n }, {\n \"name\" : \"databricksConfig.password\",\n \"value\" : null\n }, {\n \"name\" : \"databricksConfig.token\",\n \"value\" : null\n }, {\n \"name\" : \"databricksConfig.provisionNewCluster\",\n \"value\" : true\n }, {\n \"name\" : \"databricksConfig.clusterId\",\n \"value\" : null\n }, {\n \"name\" : \"databricksConfig.clusterConfig\",\n \"value\" : \"{\\n \\\"num_workers\\\": 8,\\n \\\"spark_version\\\": \\\"5.3.x-scala2.11\\\",\\n \\\"node_type_id\\\": \\\"i3.xlarge\\\"\\n}\"\n }, {\n \"name\" : \"databricksConfig.terminateCluster\",\n \"value\" : false\n }, {\n \"name\" : \"livyConfig.baseUrl\",\n \"value\" : \"https://localhost:30443/gateway/default/livy/v1/\"\n }, {\n \"name\" : \"livyConfig.username\",\n \"value\" : null\n }, {\n \"name\" : \"livyConfig.password\",\n \"value\" : null\n } ],\n \"uiInfo\" : {\n \"previewConfig\" : {\n \"previewSource\" : \"CONFIGURED_SOURCE\",\n \"batchSize\" : 10,\n \"timeout\" : 30000,\n \"writeToDestinations\" : false,\n \"executeLifecycleEvents\" : false,\n \"showHeader\" : false,\n \"showFieldType\" : true,\n \"rememberMe\" : false\n }\n },\n \"fragments\" : [ ],\n \"stages\" : [ {\n \"instanceName\" : \"AmazonS3_01\",\n \"library\" : \"streamsets-cloud-aws-lib\",\n \"stageName\" : \"com_streamsets_pipeline_stage_origin_s3_AmazonS3DSource\",\n \"stageVersion\" : \"11\",\n \"configuration\" : [ {\n \"name\" : \"s3ConfigBean.basicConfig.maxBatchSize\",\n \"value\" : 1000\n }, {\n \"name\" : \"s3ConfigBean.basicConfig.maxWaitTime\",\n \"value\" : 2000\n }, {\n \"name\" : \"s3ConfigBean.sseConfig.useCustomerSSEKey\",\n \"value\" : false\n }, {\n \"name\" : \"s3ConfigBean.sseConfig.customerKey\",\n \"value\" : null\n }, {\n \"name\" : \"s3ConfigBean.sseConfig.customerKeyMd5\",\n \"value\" : null\n }, {\n \"name\" : \"s3ConfigBean.proxyConfig.connectionTimeout\",\n \"value\" : 10\n }, {\n \"name\" : \"s3ConfigBean.proxyConfig.socketTimeout\",\n \"value\" : 50\n }, {\n \"name\" : \"s3ConfigBean.proxyConfig.retryCount\",\n \"value\" : 3\n }, {\n \"name\" : \"s3ConfigBean.proxyConfig.useProxy\",\n \"value\" : false\n }, {\n \"name\" : \"s3ConfigBean.proxyConfig.proxyHost\",\n \"value\" : null\n }, {\n \"name\" : \"s3ConfigBean.proxyConfig.proxyPort\",\n \"value\" : null\n }, {\n \"name\" : \"s3ConfigBean.proxyConfig.proxyUser\",\n \"value\" : null\n }, {\n \"name\" : \"s3ConfigBean.proxyConfig.proxyPassword\",\n \"value\" : null\n }, {\n \"name\" : \"s3ConfigBean.errorConfig.errorHandlingOption\",\n \"value\" : \"NONE\"\n }, {\n \"name\" : \"s3ConfigBean.errorConfig.archivingOption\",\n \"value\" : \"MOVE_TO_PREFIX\"\n }, {\n \"name\" : \"s3ConfigBean.errorConfig.errorPrefix\",\n \"value\" : null\n }, {\n \"name\" : \"s3ConfigBean.errorConfig.errorBucket\",\n \"value\" : null\n }, {\n \"name\" : \"s3ConfigBean.postProcessingConfig.postProcessing\",\n \"value\" : \"NONE\"\n }, {\n \"name\" : \"s3ConfigBean.postProcessingConfig.archivingOption\",\n \"value\" : \"MOVE_TO_PREFIX\"\n }, {\n \"name\" : \"s3ConfigBean.postProcessingConfig.postProcessPrefix\",\n \"value\" : null\n }, {\n \"name\" : \"s3ConfigBean.postProcessingConfig.postProcessBucket\",\n \"value\" : null\n }, {\n \"name\" : \"s3ConfigBean.s3FileConfig.prefixPattern\",\n \"value\" : \"your_file.csv\"\n }, {\n \"name\" : \"s3ConfigBean.s3FileConfig.objectOrdering\",\n \"value\" : \"LEXICOGRAPHICAL\"\n }, {\n \"name\" : \"s3ConfigBean.s3FileConfig.poolSize\",\n \"value\" : 100\n }, {\n \"name\" : \"s3ConfigBean.s3FileConfig.overrunLimit\",\n \"value\" : 128\n }, {\n \"name\" : \"s3ConfigBean.s3Config.bucket\",\n \"value\" : \"your-bucket-name\"\n }, {\n \"name\" : \"s3ConfigBean.s3Config.awsConfig.awsAccessKeyId\",\n \"value\" : \"\"\n }, {\n \"name\" : \"s3ConfigBean.s3Config.awsConfig.awsSecretAccessKey\",\n \"value\" : \"\"\n }, {\n \"name\" : \"s3ConfigBean.s3Config.region\",\n \"value\" : \"US_WEST_2\"\n }, {\n \"name\" : \"s3ConfigBean.s3Config.endpoint\",\n \"value\" : null\n }, {\n \"name\" : \"s3ConfigBean.s3Config.commonPrefix\",\n \"value\" : \"/\"\n }, {\n \"name\" : \"s3ConfigBean.s3Config.delimiter\",\n \"value\" : \"/\"\n }, {\n \"name\" : \"s3ConfigBean.enableMetaData\",\n \"value\" : false\n }, {\n \"name\" : \"s3ConfigBean.numberOfThreads\",\n \"value\" : 1\n }, {\n \"name\" : \"stageOnRecordError\",\n \"value\" : \"TO_ERROR\"\n } ],\n \"uiInfo\" : {\n \"outputStreamLabels\" : null,\n \"yPos\" : 50,\n \"stageType\" : \"SOURCE\",\n \"rawSource\" : null,\n \"icon\" : \"/packagemetadata/rest/v4/designer/icon/ae627d52-0bfc-4bc9-a67e-513bbb95753c.svg\",\n \"description\" : \"\",\n \"label\" : \"Read from S3\",\n \"xPos\" : 60\n },\n \"inputLanes\" : [ ],\n \"outputLanes\" : [ \"AmazonS3_01OutputLane15731626098910\" ],\n \"eventLanes\" : [ \"AmazonS3_01_EventLane\" ],\n \"services\" : [ {\n \"service\" : \"com.streamsets.pipeline.api.service.dataformats.DataFormatParserService\",\n \"serviceVersion\" : 1,\n \"configuration\" : [ {\n \"name\" : \"displayFormats\",\n \"value\" : \"AVRO,DELIMITED,EXCEL,JSON,LOG,PROTOBUF,SDC_JSON,TEXT,WHOLE_FILE,XML\"\n }, {\n \"name\" : \"dataFormat\",\n \"value\" : \"DELIMITED\"\n }, {\n \"name\" : \"dataFormatConfig.compression\",\n \"value\" : \"NONE\"\n }, {\n \"name\" : \"dataFormatConfig.filePatternInArchive\",\n \"value\" : \"*\"\n }, {\n \"name\" : \"dataFormatConfig.charset\",\n \"value\" : \"UTF-8\"\n }, {\n \"name\" : \"dataFormatConfig.removeCtrlChars\",\n \"value\" : false\n }, {\n \"name\" : \"dataFormatConfig.textMaxLineLen\",\n \"value\" : 1024\n }, {\n \"name\" : \"dataFormatConfig.useCustomDelimiter\",\n \"value\" : false\n }, {\n \"name\" : \"dataFormatConfig.customDelimiter\",\n \"value\" : \"\\\\r\\\\n\"\n }, {\n \"name\" : \"dataFormatConfig.includeCustomDelimiterInTheText\",\n \"value\" : false\n }, {\n \"name\" : \"dataFormatConfig.jsonContent\",\n \"value\" : \"MULTIPLE_OBJECTS\"\n }, {\n \"name\" : \"dataFormatConfig.jsonMaxObjectLen\",\n \"value\" : 4096\n }, {\n \"name\" : \"dataFormatConfig.csvFileFormat\",\n \"value\" : \"CSV\"\n }, {\n \"name\" : \"dataFormatConfig.csvHeader\",\n \"value\" : \"WITH_HEADER\"\n }, {\n \"name\" : \"dataFormatConfig.csvAllowExtraColumns\",\n \"value\" : false\n }, {\n \"name\" : \"dataFormatConfig.csvExtraColumnPrefix\",\n \"value\" : \"_extra_\"\n }, {\n \"name\" : \"dataFormatConfig.csvMaxObjectLen\",\n \"value\" : 1024\n }, {\n \"name\" : \"dataFormatConfig.csvCustomDelimiter\",\n \"value\" : \"|\"\n }, {\n \"name\" : \"dataFormatConfig.multiCharacterFieldDelimiter\",\n \"value\" : \"||\"\n }, {\n \"name\" : \"dataFormatConfig.multiCharacterLineDelimiter\",\n \"value\" : \"${str:unescapeJava('\\\\\\\\n')}\"\n }, {\n \"name\" : \"dataFormatConfig.csvCustomEscape\",\n \"value\" : \"\\\\\"\n }, {\n \"name\" : \"dataFormatConfig.csvCustomQuote\",\n \"value\" : \"\\\"\"\n }, {\n \"name\" : \"dataFormatConfig.csvEnableComments\",\n \"value\" : false\n }, {\n \"name\" : \"dataFormatConfig.csvCommentMarker\",\n \"value\" : \"#\"\n }, {\n \"name\" : \"dataFormatConfig.csvIgnoreEmptyLines\",\n \"value\" : true\n }, {\n \"name\" : \"dataFormatConfig.csvRecordType\",\n \"value\" : \"LIST_MAP\"\n }, {\n \"name\" : \"dataFormatConfig.csvSkipStartLines\",\n \"value\" : 0\n }, {\n \"name\" : \"dataFormatConfig.parseNull\",\n \"value\" : false\n }, {\n \"name\" : \"dataFormatConfig.nullConstant\",\n \"value\" : \"\\\\\\\\N\"\n }, {\n \"name\" : \"dataFormatConfig.xmlRecordElement\",\n \"value\" : null\n }, {\n \"name\" : \"dataFormatConfig.includeFieldXpathAttributes\",\n \"value\" : false\n }, {\n \"name\" : \"dataFormatConfig.xPathNamespaceContext\",\n \"value\" : [ ]\n }, {\n \"name\" : \"dataFormatConfig.outputFieldAttributes\",\n \"value\" : false\n }, {\n \"name\" : \"dataFormatConfig.xmlMaxObjectLen\",\n \"value\" : 4096\n }, {\n \"name\" : \"dataFormatConfig.logMode\",\n \"value\" : \"COMMON_LOG_FORMAT\"\n }, {\n \"name\" : \"dataFormatConfig.logMaxObjectLen\",\n \"value\" : 1024\n }, {\n \"name\" : \"dataFormatConfig.retainOriginalLine\",\n \"value\" : false\n }, {\n \"name\" : \"dataFormatConfig.customLogFormat\",\n \"value\" : \"%h %l %u %t \\\"%r\\\" %>s %b\"\n }, {\n \"name\" : \"dataFormatConfig.regex\",\n \"value\" : \"^(\\\\S+) (\\\\S+) (\\\\S+) \\\\[([\\\\w:/]+\\\\s[+\\\\-]\\\\d{4})\\\\] \\\"(\\\\S+) (\\\\S+) (\\\\S+)\\\" (\\\\d{3}) (\\\\d+)\"\n }, {\n \"name\" : \"dataFormatConfig.fieldPathsToGroupName\",\n \"value\" : [ {\n \"fieldPath\" : \"/\",\n \"group\" : 1\n } ]\n }, {\n \"name\" : \"dataFormatConfig.grokPatternDefinition\",\n \"value\" : null\n }, {\n \"name\" : \"dataFormatConfig.grokPattern\",\n \"value\" : \"%{COMMONAPACHELOG}\"\n }, {\n \"name\" : \"dataFormatConfig.onParseError\",\n \"value\" : \"ERROR\"\n }, {\n \"name\" : \"dataFormatConfig.maxStackTraceLines\",\n \"value\" : 50\n }, {\n \"name\" : \"dataFormatConfig.enableLog4jCustomLogFormat\",\n \"value\" : false\n }, {\n \"name\" : \"dataFormatConfig.log4jCustomLogFormat\",\n \"value\" : \"%r [%t] %-5p %c %x - %m%n\"\n }, {\n \"name\" : \"dataFormatConfig.avroSchemaSource\",\n \"value\" : null\n }, {\n \"name\" : \"dataFormatConfig.avroSchema\",\n \"value\" : null\n }, {\n \"name\" : \"dataFormatConfig.schemaRegistryUrls\",\n \"value\" : [ ]\n }, {\n \"name\" : \"dataFormatConfig.basicAuth\",\n \"value\" : null\n }, {\n \"name\" : \"dataFormatConfig.schemaLookupMode\",\n \"value\" : \"SUBJECT\"\n }, {\n \"name\" : \"dataFormatConfig.subject\",\n \"value\" : null\n }, {\n \"name\" : \"dataFormatConfig.schemaId\",\n \"value\" : null\n }, {\n \"name\" : \"dataFormatConfig.avroSkipUnionIndex\",\n \"value\" : false\n }, {\n \"name\" : \"dataFormatConfig.protoDescriptorFile\",\n \"value\" : null\n }, {\n \"name\" : \"dataFormatConfig.messageType\",\n \"value\" : null\n }, {\n \"name\" : \"dataFormatConfig.isDelimited\",\n \"value\" : true\n }, {\n \"name\" : \"dataFormatConfig.binaryMaxObjectLen\",\n \"value\" : 1024\n }, {\n \"name\" : \"dataFormatConfig.datagramMode\",\n \"value\" : \"SYSLOG\"\n }, {\n \"name\" : \"dataFormatConfig.typesDbPath\",\n \"value\" : null\n }, {\n \"name\" : \"dataFormatConfig.convertTime\",\n \"value\" : false\n }, {\n \"name\" : \"dataFormatConfig.excludeInterval\",\n \"value\" : true\n }, {\n \"name\" : \"dataFormatConfig.authFilePath\",\n \"value\" : null\n }, {\n \"name\" : \"dataFormatConfig.netflowOutputValuesMode\",\n \"value\" : \"RAW_AND_INTERPRETED\"\n }, {\n \"name\" : \"dataFormatConfig.maxTemplateCacheSize\",\n \"value\" : -1\n }, {\n \"name\" : \"dataFormatConfig.templateCacheTimeoutMs\",\n \"value\" : -1\n }, {\n \"name\" : \"dataFormatConfig.netflowOutputValuesModeDatagram\",\n \"value\" : \"RAW_AND_INTERPRETED\"\n }, {\n \"name\" : \"dataFormatConfig.maxTemplateCacheSizeDatagram\",\n \"value\" : -1\n }, {\n \"name\" : \"dataFormatConfig.templateCacheTimeoutMsDatagram\",\n \"value\" : -1\n }, {\n \"name\" : \"dataFormatConfig.wholeFileMaxObjectLen\",\n \"value\" : 8192\n }, {\n \"name\" : \"dataFormatConfig.rateLimit\",\n \"value\" : \"-1\"\n }, {\n \"name\" : \"dataFormatConfig.verifyChecksum\",\n \"value\" : false\n }, {\n \"name\" : \"dataFormatConfig.excelHeader\",\n \"value\" : null\n }, {\n \"name\" : \"dataFormatConfig.excelSkipCellsWithNoHeader\",\n \"value\" : false\n }, {\n \"name\" : \"dataFormatConfig.excelReadAllSheets\",\n \"value\" : true\n }, {\n \"name\" : \"dataFormatConfig.excelSheetNames\",\n \"value\" : [ ]\n } ]\n } ]\n }, {\n \"instanceName\" : \"StreamSelector_01\",\n \"library\" : \"streamsets-cloud-basic-lib\",\n \"stageName\" : \"com_streamsets_pipeline_stage_processor_selector_SelectorDProcessor\",\n \"stageVersion\" : \"1\",\n \"configuration\" : [ {\n \"name\" : \"lanePredicates\",\n \"value\" : [ {\n \"outputLane\" : \"StreamSelector_01OutputLane1573162683263\",\n \"predicate\" : \"${record:value('/payment_type')=='CRD'}\"\n }, {\n \"outputLane\" : \"StreamSelector_01OutputLane1573162629916\",\n \"predicate\" : \"default\"\n } ]\n }, {\n \"name\" : \"stageOnRecordError\",\n \"value\" : \"TO_ERROR\"\n }, {\n \"name\" : \"stageRequiredFields\",\n \"value\" : [ ]\n }, {\n \"name\" : \"stageRecordPreconditions\",\n \"value\" : [ ]\n } ],\n \"uiInfo\" : {\n \"outputStreamLabels\" : null,\n \"yPos\" : 50,\n \"stageType\" : \"PROCESSOR\",\n \"rawSource\" : null,\n \"icon\" : \"/packagemetadata/rest/v4/designer/icon/68d5efa5-2d88-4550-819a-2cf14635fc24.svg\",\n \"description\" : \"\",\n \"label\" : \"Filter credit card txns\",\n \"xPos\" : 280\n },\n \"inputLanes\" : [ \"AmazonS3_01OutputLane15731626098910\" ],\n \"outputLanes\" : [ \"StreamSelector_01OutputLane1573162683263\", \"StreamSelector_01OutputLane1573162629916\" ],\n \"eventLanes\" : [ ],\n \"services\" : [ ]\n }, {\n \"instanceName\" : \"FieldRemover_01\",\n \"library\" : \"streamsets-cloud-basic-lib\",\n \"stageName\" : \"com_streamsets_pipeline_stage_processor_fieldfilter_FieldFilterDProcessor\",\n \"stageVersion\" : \"1\",\n \"configuration\" : [ {\n \"name\" : \"filterOperation\",\n \"value\" : \"REMOVE\"\n }, {\n \"name\" : \"fields\",\n \"value\" : [ \"/credit_card\" ]\n }, {\n \"name\" : \"constant\",\n \"value\" : null\n }, {\n \"name\" : \"stageRequiredFields\",\n \"value\" : [ ]\n }, {\n \"name\" : \"stageRecordPreconditions\",\n \"value\" : [ ]\n } ],\n \"uiInfo\" : {\n \"outputStreamLabels\" : null,\n \"yPos\" : 50,\n \"stageType\" : \"PROCESSOR\",\n \"rawSource\" : null,\n \"icon\" : \"/packagemetadata/rest/v4/designer/icon/ff45e99a-90a8-4a84-8173-1c99c79f71c3.svg\",\n \"description\" : \"\",\n \"label\" : \"Remove credit card number\",\n \"xPos\" : 500\n },\n \"inputLanes\" : [ \"StreamSelector_01OutputLane1573162683263\" ],\n \"outputLanes\" : [ \"FieldRemover_01OutputLane15731626396560\" ],\n \"eventLanes\" : [ ],\n \"services\" : [ ]\n }, {\n \"instanceName\" : \"FieldTypeConverter_01\",\n \"library\" : \"streamsets-cloud-basic-lib\",\n \"stageName\" : \"com_streamsets_pipeline_stage_processor_fieldtypeconverter_FieldTypeConverterDProcessor\",\n \"stageVersion\" : \"2\",\n \"configuration\" : [ {\n \"name\" : \"convertBy\",\n \"value\" : \"BY_FIELD\"\n }, {\n \"name\" : \"fieldTypeConverterConfigs\",\n \"value\" : [ {\n \"fields\" : [ \"/fare_amount\", \"/surcharge\", \"/mta_tax\", \"/tip_amount\", \"/tolls_amount\", \"/total_amount\", \"/trip_distance\", \"/pickup_longitude\", \"/pickup_latitude\", \"/dropoff_longitude\", \"/dropoff_latitude\" ],\n \"targetType\" : \"FLOAT\",\n \"treatInputFieldAsDate\" : false,\n \"dataLocale\" : \"en,US\",\n \"scale\" : 2,\n \"decimalScaleRoundingStrategy\" : \"ROUND_UNNECESSARY\",\n \"dateFormat\" : \"YYYY_MM_DD\",\n \"zonedDateTimeFormat\" : \"ISO_ZONED_DATE_TIME\",\n \"encoding\" : \"UTF-8\"\n }, {\n \"fields\" : [ \"/rate_code\", \"/passenger_count\", \"/trip_time_in_secs\" ],\n \"targetType\" : \"INTEGER\",\n \"treatInputFieldAsDate\" : false,\n \"dataLocale\" : \"en,US\",\n \"scale\" : -1,\n \"decimalScaleRoundingStrategy\" : \"ROUND_UNNECESSARY\",\n \"dateFormat\" : \"YYYY_MM_DD\",\n \"zonedDateTimeFormat\" : \"ISO_ZONED_DATE_TIME\",\n \"encoding\" : \"UTF-8\"\n }, {\n \"fields\" : [ \"/pickup_datetime\", \"/dropoff_datetime\" ],\n \"targetType\" : \"DATETIME\",\n \"treatInputFieldAsDate\" : false,\n \"dataLocale\" : \"en,US\",\n \"scale\" : -1,\n \"decimalScaleRoundingStrategy\" : \"ROUND_UNNECESSARY\",\n \"dateFormat\" : \"YYYY_MM_DD_HH_MM_SS\",\n \"zonedDateTimeFormat\" : \"ISO_ZONED_DATE_TIME\",\n \"encoding\" : \"UTF-8\"\n } ]\n }, {\n \"name\" : \"wholeTypeConverterConfigs\",\n \"value\" : [ {\n \"sourceType\" : \"INTEGER\",\n \"targetType\" : \"INTEGER\",\n \"treatInputFieldAsDate\" : false,\n \"dataLocale\" : \"en,US\",\n \"scale\" : -1,\n \"decimalScaleRoundingStrategy\" : \"ROUND_UNNECESSARY\",\n \"dateFormat\" : \"YYYY_MM_DD\",\n \"zonedDateTimeFormat\" : \"ISO_ZONED_DATE_TIME\",\n \"encoding\" : \"UTF-8\"\n } ]\n }, {\n \"name\" : \"stageOnRecordError\",\n \"value\" : \"TO_ERROR\"\n }, {\n \"name\" : \"stageRequiredFields\",\n \"value\" : [ ]\n }, {\n \"name\" : \"stageRecordPreconditions\",\n \"value\" : [ ]\n } ],\n \"uiInfo\" : {\n \"outputStreamLabels\" : null,\n \"yPos\" : 50,\n \"stageType\" : \"PROCESSOR\",\n \"rawSource\" : null,\n \"icon\" : \"/packagemetadata/rest/v4/designer/icon/8fabea30-1f37-4a75-954e-21d630439710.svg\",\n \"description\" : \"\",\n \"label\" : \"Convert field types\",\n \"xPos\" : 720\n },\n \"inputLanes\" : [ \"FieldRemover_01OutputLane15731626396560\" ],\n \"outputLanes\" : [ \"FieldTypeConverter_01OutputLane15731626513390\" ],\n \"eventLanes\" : [ ],\n \"services\" : [ ]\n }, {\n \"instanceName\" : \"AzureSQLDataWarehouseDestination_01\",\n \"library\" : \"streamsets-cloud-azure-lib\",\n \"stageName\" : \"com_streamsets_pipeline_stage_destination_datawarehouse_AzureDataWarehouseDTarget\",\n \"stageVersion\" : \"3\",\n \"configuration\" : [ {\n \"name\" : \"config.dataWarehouseConfig.server\",\n \"value\" : \"your-data-warehouse\"\n }, {\n \"name\" : \"config.dataWarehouseConfig.database\",\n \"value\" : \"your-database\"\n }, {\n \"name\" : \"config.dataWarehouseConfig.authMethod\",\n \"value\" : \"SQL_SERVER_LOGIN\"\n }, {\n \"name\" : \"config.dataWarehouseConfig.user\",\n \"value\" : \"\"\n }, {\n \"name\" : \"config.dataWarehouseConfig.password\",\n \"value\" : \"\"\n }, {\n \"name\" : \"config.dataWarehouseConfig.aadUserId\",\n \"value\" : null\n }, {\n \"name\" : \"config.dataWarehouseConfig.aadPassword\",\n \"value\" : null\n }, {\n \"name\" : \"config.dataWarehouseConfig.connectionPoolSize\",\n \"value\" : 0\n }, {\n \"name\" : \"config.dataWarehouseConfig.tableConfig.schema\",\n \"value\" : \"your-schema\"\n }, {\n \"name\" : \"config.dataWarehouseConfig.tableConfig.destinationTable\",\n \"value\" : \"taxi\"\n }, {\n \"name\" : \"config.dataWarehouseConfig.tableConfig.autoCreateTable\",\n \"value\" : true\n }, {\n \"name\" : \"config.dataWarehouseConfig.tableConfig.dataDriftEnabled\",\n \"value\" : true\n }, {\n \"name\" : \"config.dataWarehouseConfig.tableConfig.alwaysCreateVarcharColumns\",\n \"value\" : false\n }, {\n \"name\" : \"config.dataWarehouseConfig.tableConfig.ignoreMissingFields\",\n \"value\" : true\n }, {\n \"name\" : \"config.dataWarehouseConfig.tableConfig.ignoreFieldsWithInvalidTypes\",\n \"value\" : true\n }, {\n \"name\" : \"config.dataWarehouseConfig.tableConfig.caseInsensitive\",\n \"value\" : true\n }, {\n \"name\" : \"config.azureStage.accountName\",\n \"value\" : \"\"\n }, {\n \"name\" : \"config.azureStage.storageContainer\",\n \"value\" : \"your-container\"\n }, {\n \"name\" : \"config.azureStage.sasToken\",\n \"value\" : \"\"\n }, {\n \"name\" : \"config.azureStage.authMethod\",\n \"value\" : \"SHARED_KEY\"\n }, {\n \"name\" : \"config.azureStage.clientId\",\n \"value\" : null\n }, {\n \"name\" : \"config.azureStage.authTokenEndpoint\",\n \"value\" : \"https://login.microsoftonline.com/example-example\"\n }, {\n \"name\" : \"config.azureStage.clientKey\",\n \"value\" : null\n }, {\n \"name\" : \"config.azureStage.accountKey\",\n \"value\" : \"\"\n }, {\n \"name\" : \"config.azureStage.filePrefix\",\n \"value\" : \"your-prefix\"\n }, {\n \"name\" : \"config.azureStage.purgeDataFile\",\n \"value\" : false\n }, {\n \"name\" : \"config.dataConfigs.rowFieldPath\",\n \"value\" : \"/\"\n }, {\n \"name\" : \"config.dataConfigs.ignoreFields\",\n \"value\" : null\n }, {\n \"name\" : \"config.dataConfigs.booleanDefault\",\n \"value\" : null\n }, {\n \"name\" : \"config.dataConfigs.numberDefault\",\n \"value\" : null\n }, {\n \"name\" : \"config.dataConfigs.dateDefault\",\n \"value\" : null\n }, {\n \"name\" : \"config.dataConfigs.timeDefault\",\n \"value\" : null\n }, {\n \"name\" : \"config.dataConfigs.dateTimeDefault\",\n \"value\" : null\n }, {\n \"name\" : \"config.dataConfigs.varcharDefault\",\n \"value\" : null\n }, {\n \"name\" : \"config.dataConfigs.binaryDefault\",\n \"value\" : null\n }, {\n \"name\" : \"stageOnRecordError\",\n \"value\" : \"TO_ERROR\"\n }, {\n \"name\" : \"stageRequiredFields\",\n \"value\" : [ ]\n }, {\n \"name\" : \"stageRecordPreconditions\",\n \"value\" : [ ]\n } ],\n \"uiInfo\" : {\n \"outputStreamLabels\" : null,\n \"yPos\" : 50,\n \"stageType\" : \"TARGET\",\n \"rawSource\" : null,\n \"icon\" : \"/packagemetadata/rest/v4/designer/icon/b7c84256-76d6-4a68-b5ad-cf535392f8ea.png\",\n \"description\" : \"\",\n \"label\" : \"Azure SQL Data Warehouse Destination 1\",\n \"xPos\" : 940\n },\n \"inputLanes\" : [ \"FieldTypeConverter_01OutputLane15731626513390\" ],\n \"outputLanes\" : [ ],\n \"eventLanes\" : [ ],\n \"services\" : [ ]\n }, {\n \"instanceName\" : \"Trash_01\",\n \"library\" : \"streamsets-cloud-basic-lib\",\n \"stageName\" : \"com_streamsets_pipeline_stage_destination_devnull_NullDTarget\",\n \"stageVersion\" : \"1\",\n \"configuration\" : [ ],\n \"uiInfo\" : {\n \"outputStreamLabels\" : null,\n \"yPos\" : 180,\n \"stageType\" : \"TARGET\",\n \"rawSource\" : null,\n \"icon\" : \"/packagemetadata/rest/v4/designer/icon/b182097c-9d4a-4e46-97a8-51ba408a2055.svg\",\n \"description\" : \"\",\n \"label\" : \"Discard non cc txns\",\n \"xPos\" : 500\n },\n \"inputLanes\" : [ \"StreamSelector_01OutputLane1573162629916\" ],\n \"outputLanes\" : [ ],\n \"eventLanes\" : [ ],\n \"services\" : [ ]\n }, {\n \"instanceName\" : \"PipelineFinisherExecutor_01\",\n \"library\" : \"streamsets-cloud-basic-lib\",\n \"stageName\" : \"com_streamsets_pipeline_stage_executor_finishpipeline_PipelineFinisherDExecutor\",\n \"stageVersion\" : \"1\",\n \"configuration\" : [ {\n \"name\" : \"config.resetOffset\",\n \"value\" : false\n }, {\n \"name\" : \"stageOnRecordError\",\n \"value\" : \"DISCARD\"\n }, {\n \"name\" : \"stageRequiredFields\",\n \"value\" : [ ]\n }, {\n \"name\" : \"stageRecordPreconditions\",\n \"value\" : [ \"${record:eventType() == 'no-more-data'}\" ]\n } ],\n \"uiInfo\" : {\n \"outputStreamLabels\" : null,\n \"yPos\" : 180,\n \"stageType\" : \"EXECUTOR\",\n \"rawSource\" : null,\n \"icon\" : \"/packagemetadata/rest/v4/designer/icon/8199d068-c2d7-406a-a2d2-91985534832b.png\",\n \"description\" : \"\",\n \"label\" : \"Stop when no more data\",\n \"xPos\" : 280\n },\n \"inputLanes\" : [ \"AmazonS3_01_EventLane\" ],\n \"outputLanes\" : [ ],\n \"eventLanes\" : [ ],\n \"services\" : [ ]\n } ],\n \"errorStage\" : {\n \"instanceName\" : \"Discard_ErrorStage\",\n \"library\" : \"streamsets-cloud-basic-lib\",\n \"stageName\" : \"com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget\",\n \"stageVersion\" : \"1\",\n \"configuration\" : [ ],\n \"uiInfo\" : {\n \"outputStreamLabels\" : null,\n \"yPos\" : 50,\n \"stageType\" : \"TARGET\",\n \"rawSource\" : null,\n \"icon\" : null,\n \"description\" : \"\",\n \"label\" : \"Error Records - Discard\",\n \"xPos\" : 60\n },\n \"inputLanes\" : [ ],\n \"outputLanes\" : [ ],\n \"eventLanes\" : [ ],\n \"services\" : [ ]\n },\n \"info\" : {\n \"pipelineId\" : \"retertfe48397d-0ee4-450c-a7f2-6a40ba666a73\",\n \"title\" : \"retert\",\n \"description\" : \"ertret\",\n \"created\" : 1554928504895,\n \"lastModified\" : 1554928504895,\n \"creator\" : \"admin\",\n \"lastModifier\" : \"admin\",\n \"lastRev\" : \"0\",\n \"uuid\" : \"c43e05e5-b890-4bab-a8a3-69432b244641\",\n \"valid\" : false,\n \"metadata\" : null,\n \"name\" : \"retertfe48397d-0ee4-450c-a7f2-6a40ba666a73\",\n \"sdcVersion\" : \"3.8.1\",\n \"sdcId\" : \"771a89c4-5732-11e9-b037-c3eca82085d1\"\n },\n \"metadata\" : {\n \"stageLibraryVersions\" : {\n \"streamsets-cloud-aws-lib\" : \"1.2.0\",\n \"streamsets-cloud-basic-lib\" : \"1.4.0\",\n \"streamsets-cloud-azure-lib\" : \"1.5.1\"\n },\n \"executorVersion\" : \"1.2.0\",\n \"productType\" : \"StreamSets Cloud\"\n },\n \"statsAggregatorStage\" : null,\n \"startEventStages\" : [ ],\n \"stopEventStages\" : [ ],\n \"testOriginStage\" : null,\n \"valid\" : true,\n \"issues\" : {\n \"pipelineIssues\" : [ ],\n \"stageIssues\" : { },\n \"issueCount\" : 0\n },\n \"previewable\" : true\n}",
"pipelineLabels" : [ ],
"provenanceMetaData" : { },
"sdcVersion" : "3.8.1",
"system" : false,
"designMode" : "BASIC",
"name" : "S3 to Azure Synapse",
"vaultId" : "9caa30cf-95d5-40fc-b0f7-d7e2c08add32"
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment