Created
September 7, 2023 19:28
-
-
Save mattyb149/0328489ce0d37b51d3b204c41f399b40 to your computer and use it in GitHub Desktop.
NiFi 1.23+ Flow Definition to extract field names from schema
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "flowContents": { | |
| "identifier": "a226434e-78d6-3846-bb1f-55680679a3b3", | |
| "instanceIdentifier": "710e0519-018a-1000-072b-2e0e390412d3", | |
| "name": "NiFi Flow", | |
| "comments": "", | |
| "position": { | |
| "x": 0.0, | |
| "y": 0.0 | |
| }, | |
| "processGroups": [], | |
| "remoteProcessGroups": [], | |
| "processors": [{ | |
| "identifier": "328bb2df-eecc-33c4-b761-911d7559861a", | |
| "instanceIdentifier": "7112f79b-018a-1000-8d03-6ed45300a290", | |
| "name": "UpdateAttribute", | |
| "comments": "", | |
| "position": { | |
| "x": 392.0, | |
| "y": 392.0 | |
| }, | |
| "type": "org.apache.nifi.processors.attributes.UpdateAttribute", | |
| "bundle": { | |
| "group": "org.apache.nifi", | |
| "artifact": "nifi-update-attribute-nar", | |
| "version": "2.0.0-SNAPSHOT" | |
| }, | |
| "properties": { | |
| "Delete Attributes Expression": null, | |
| "Store State": "Do not store state", | |
| "canonical-value-lookup-cache-size": "100", | |
| "Stateful Variables Initial Value": null, | |
| "fields": "${avro.schema:jsonPath('$.fields[*].name'):replaceAll('[\\\"\\[\\]]','')}" | |
| }, | |
| "propertyDescriptors": { | |
| "Delete Attributes Expression": { | |
| "name": "Delete Attributes Expression", | |
| "displayName": "Delete Attributes Expression", | |
| "identifiesControllerService": false, | |
| "sensitive": false, | |
| "dynamic": false | |
| }, | |
| "Store State": { | |
| "name": "Store State", | |
| "displayName": "Store State", | |
| "identifiesControllerService": false, | |
| "sensitive": false, | |
| "dynamic": false | |
| }, | |
| "canonical-value-lookup-cache-size": { | |
| "name": "canonical-value-lookup-cache-size", | |
| "displayName": "Cache Value Lookup Cache Size", | |
| "identifiesControllerService": false, | |
| "sensitive": false, | |
| "dynamic": false | |
| }, | |
| "Stateful Variables Initial Value": { | |
| "name": "Stateful Variables Initial Value", | |
| "displayName": "Stateful Variables Initial Value", | |
| "identifiesControllerService": false, | |
| "sensitive": false, | |
| "dynamic": false | |
| }, | |
| "fields": { | |
| "name": "fields", | |
| "displayName": "fields", | |
| "identifiesControllerService": false, | |
| "sensitive": false, | |
| "dynamic": true | |
| } | |
| }, | |
| "style": {}, | |
| "schedulingPeriod": "0 sec", | |
| "schedulingStrategy": "TIMER_DRIVEN", | |
| "executionNode": "ALL", | |
| "penaltyDuration": "30 sec", | |
| "yieldDuration": "1 sec", | |
| "bulletinLevel": "WARN", | |
| "runDurationMillis": 25, | |
| "concurrentlySchedulableTaskCount": 1, | |
| "autoTerminatedRelationships": [], | |
| "scheduledState": "ENABLED", | |
| "retryCount": 10, | |
| "retriedRelationships": [], | |
| "backoffMechanism": "PENALIZE_FLOWFILE", | |
| "maxBackoffPeriod": "10 mins", | |
| "componentType": "PROCESSOR", | |
| "groupIdentifier": "a226434e-78d6-3846-bb1f-55680679a3b3" | |
| }, { | |
| "identifier": "4ab8f025-2011-398d-b224-cd543224b4f6", | |
| "instanceIdentifier": "71101eee-018a-1000-f48c-611f43c32ef3", | |
| "name": "ExtractRecordSchema", | |
| "comments": "", | |
| "position": { | |
| "x": 392.0, | |
| "y": 184.0 | |
| }, | |
| "type": "org.apache.nifi.processors.standard.ExtractRecordSchema", | |
| "bundle": { | |
| "group": "org.apache.nifi", | |
| "artifact": "nifi-standard-nar", | |
| "version": "2.0.0-SNAPSHOT" | |
| }, | |
| "properties": { | |
| "cache-size": "10", | |
| "record-reader": "7af764bf-e0f7-31eb-ad0d-eacbe7d05b1e" | |
| }, | |
| "propertyDescriptors": { | |
| "cache-size": { | |
| "name": "cache-size", | |
| "displayName": "Schema Cache Size", | |
| "identifiesControllerService": false, | |
| "sensitive": false, | |
| "dynamic": false | |
| }, | |
| "record-reader": { | |
| "name": "record-reader", | |
| "displayName": "Record Reader", | |
| "identifiesControllerService": true, | |
| "sensitive": false, | |
| "dynamic": false | |
| } | |
| }, | |
| "style": {}, | |
| "schedulingPeriod": "0 sec", | |
| "schedulingStrategy": "TIMER_DRIVEN", | |
| "executionNode": "ALL", | |
| "penaltyDuration": "30 sec", | |
| "yieldDuration": "1 sec", | |
| "bulletinLevel": "WARN", | |
| "runDurationMillis": 0, | |
| "concurrentlySchedulableTaskCount": 1, | |
| "autoTerminatedRelationships": ["failure"], | |
| "scheduledState": "ENABLED", | |
| "retryCount": 10, | |
| "retriedRelationships": [], | |
| "backoffMechanism": "PENALIZE_FLOWFILE", | |
| "maxBackoffPeriod": "10 mins", | |
| "componentType": "PROCESSOR", | |
| "groupIdentifier": "a226434e-78d6-3846-bb1f-55680679a3b3" | |
| }, { | |
| "identifier": "a40198d5-632c-3d65-b0e6-a583b23aa065", | |
| "instanceIdentifier": "710ffcc8-018a-1000-2451-9dff878f8140", | |
| "name": "GenerateFlowFile", | |
| "comments": "", | |
| "position": { | |
| "x": 392.0, | |
| "y": -24.0 | |
| }, | |
| "type": "org.apache.nifi.processors.standard.GenerateFlowFile", | |
| "bundle": { | |
| "group": "org.apache.nifi", | |
| "artifact": "nifi-standard-nar", | |
| "version": "2.0.0-SNAPSHOT" | |
| }, | |
| "properties": { | |
| "character-set": "UTF-8", | |
| "File Size": "0B", | |
| "mime-type": "text/csv", | |
| "generate-ff-custom-text": "id,x,y\n1,Hello,2.0\n2,World,3.0", | |
| "Batch Size": "1", | |
| "Unique FlowFiles": "false", | |
| "Data Format": "Text" | |
| }, | |
| "propertyDescriptors": { | |
| "character-set": { | |
| "name": "character-set", | |
| "displayName": "Character Set", | |
| "identifiesControllerService": false, | |
| "sensitive": false, | |
| "dynamic": false | |
| }, | |
| "File Size": { | |
| "name": "File Size", | |
| "displayName": "File Size", | |
| "identifiesControllerService": false, | |
| "sensitive": false, | |
| "dynamic": false | |
| }, | |
| "mime-type": { | |
| "name": "mime-type", | |
| "displayName": "Mime Type", | |
| "identifiesControllerService": false, | |
| "sensitive": false, | |
| "dynamic": false | |
| }, | |
| "generate-ff-custom-text": { | |
| "name": "generate-ff-custom-text", | |
| "displayName": "Custom Text", | |
| "identifiesControllerService": false, | |
| "sensitive": false, | |
| "dynamic": false | |
| }, | |
| "Batch Size": { | |
| "name": "Batch Size", | |
| "displayName": "Batch Size", | |
| "identifiesControllerService": false, | |
| "sensitive": false, | |
| "dynamic": false | |
| }, | |
| "Unique FlowFiles": { | |
| "name": "Unique FlowFiles", | |
| "displayName": "Unique FlowFiles", | |
| "identifiesControllerService": false, | |
| "sensitive": false, | |
| "dynamic": false | |
| }, | |
| "Data Format": { | |
| "name": "Data Format", | |
| "displayName": "Data Format", | |
| "identifiesControllerService": false, | |
| "sensitive": false, | |
| "dynamic": false | |
| } | |
| }, | |
| "style": {}, | |
| "schedulingPeriod": "1 min", | |
| "schedulingStrategy": "TIMER_DRIVEN", | |
| "executionNode": "ALL", | |
| "penaltyDuration": "30 sec", | |
| "yieldDuration": "1 sec", | |
| "bulletinLevel": "WARN", | |
| "runDurationMillis": 0, | |
| "concurrentlySchedulableTaskCount": 1, | |
| "autoTerminatedRelationships": [], | |
| "scheduledState": "ENABLED", | |
| "retryCount": 10, | |
| "retriedRelationships": [], | |
| "backoffMechanism": "PENALIZE_FLOWFILE", | |
| "maxBackoffPeriod": "10 mins", | |
| "componentType": "PROCESSOR", | |
| "groupIdentifier": "a226434e-78d6-3846-bb1f-55680679a3b3" | |
| }], | |
| "inputPorts": [], | |
| "outputPorts": [], | |
| "connections": [{ | |
| "identifier": "54d53ebe-6ebb-3691-8a10-61f4231b2c14", | |
| "instanceIdentifier": "711048cd-018a-1000-a3dd-f899183314bd", | |
| "name": "", | |
| "source": { | |
| "id": "a40198d5-632c-3d65-b0e6-a583b23aa065", | |
| "type": "PROCESSOR", | |
| "groupId": "a226434e-78d6-3846-bb1f-55680679a3b3", | |
| "name": "GenerateFlowFile", | |
| "comments": "", | |
| "instanceIdentifier": "710ffcc8-018a-1000-2451-9dff878f8140" | |
| }, | |
| "destination": { | |
| "id": "4ab8f025-2011-398d-b224-cd543224b4f6", | |
| "type": "PROCESSOR", | |
| "groupId": "a226434e-78d6-3846-bb1f-55680679a3b3", | |
| "name": "ExtractRecordSchema", | |
| "comments": "", | |
| "instanceIdentifier": "71101eee-018a-1000-f48c-611f43c32ef3" | |
| }, | |
| "labelIndex": 1, | |
| "zIndex": 0, | |
| "selectedRelationships": ["success"], | |
| "backPressureObjectThreshold": 10000, | |
| "backPressureDataSizeThreshold": "1 GB", | |
| "flowFileExpiration": "0 sec", | |
| "prioritizers": [], | |
| "bends": [], | |
| "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", | |
| "partitioningAttribute": "", | |
| "loadBalanceCompression": "DO_NOT_COMPRESS", | |
| "componentType": "CONNECTION", | |
| "groupIdentifier": "a226434e-78d6-3846-bb1f-55680679a3b3" | |
| }, { | |
| "identifier": "fa781453-134d-3411-9d04-16792f66d97c", | |
| "instanceIdentifier": "711331c3-018a-1000-bf0a-cd2ac287a48d", | |
| "name": "", | |
| "source": { | |
| "id": "328bb2df-eecc-33c4-b761-911d7559861a", | |
| "type": "PROCESSOR", | |
| "groupId": "a226434e-78d6-3846-bb1f-55680679a3b3", | |
| "name": "UpdateAttribute", | |
| "comments": "", | |
| "instanceIdentifier": "7112f79b-018a-1000-8d03-6ed45300a290" | |
| }, | |
| "destination": { | |
| "id": "e8ea588a-1161-3123-ab4f-e17f4e2f54f7", | |
| "type": "FUNNEL", | |
| "groupId": "a226434e-78d6-3846-bb1f-55680679a3b3", | |
| "name": "Funnel", | |
| "comments": "", | |
| "instanceIdentifier": "71102e60-018a-1000-9d2e-0ba802462681" | |
| }, | |
| "labelIndex": 1, | |
| "zIndex": 0, | |
| "selectedRelationships": ["success"], | |
| "backPressureObjectThreshold": 10000, | |
| "backPressureDataSizeThreshold": "1 GB", | |
| "flowFileExpiration": "0 sec", | |
| "prioritizers": [], | |
| "bends": [], | |
| "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", | |
| "partitioningAttribute": "", | |
| "loadBalanceCompression": "DO_NOT_COMPRESS", | |
| "componentType": "CONNECTION", | |
| "groupIdentifier": "a226434e-78d6-3846-bb1f-55680679a3b3" | |
| }, { | |
| "identifier": "d75b4a34-394a-31fa-84d4-3e10f51b4a9a", | |
| "instanceIdentifier": "7110ba7b-018a-1000-260e-191930d3995e", | |
| "name": "", | |
| "source": { | |
| "id": "4ab8f025-2011-398d-b224-cd543224b4f6", | |
| "type": "PROCESSOR", | |
| "groupId": "a226434e-78d6-3846-bb1f-55680679a3b3", | |
| "name": "ExtractRecordSchema", | |
| "comments": "", | |
| "instanceIdentifier": "71101eee-018a-1000-f48c-611f43c32ef3" | |
| }, | |
| "destination": { | |
| "id": "328bb2df-eecc-33c4-b761-911d7559861a", | |
| "type": "PROCESSOR", | |
| "groupId": "a226434e-78d6-3846-bb1f-55680679a3b3", | |
| "name": "UpdateAttribute", | |
| "comments": "", | |
| "instanceIdentifier": "7112f79b-018a-1000-8d03-6ed45300a290" | |
| }, | |
| "labelIndex": 1, | |
| "zIndex": 0, | |
| "selectedRelationships": ["success"], | |
| "backPressureObjectThreshold": 10000, | |
| "backPressureDataSizeThreshold": "1 GB", | |
| "flowFileExpiration": "0 sec", | |
| "prioritizers": [], | |
| "bends": [], | |
| "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", | |
| "partitioningAttribute": "", | |
| "loadBalanceCompression": "DO_NOT_COMPRESS", | |
| "componentType": "CONNECTION", | |
| "groupIdentifier": "a226434e-78d6-3846-bb1f-55680679a3b3" | |
| }], | |
| "labels": [], | |
| "funnels": [{ | |
| "identifier": "e8ea588a-1161-3123-ab4f-e17f4e2f54f7", | |
| "instanceIdentifier": "71102e60-018a-1000-9d2e-0ba802462681", | |
| "position": { | |
| "x": 544.0, | |
| "y": 600.0 | |
| }, | |
| "componentType": "FUNNEL", | |
| "groupIdentifier": "a226434e-78d6-3846-bb1f-55680679a3b3" | |
| }], | |
| "controllerServices": [{ | |
| "identifier": "7af764bf-e0f7-31eb-ad0d-eacbe7d05b1e", | |
| "instanceIdentifier": "7110e6ee-018a-1000-f994-07e9d88ccb99", | |
| "name": "CSVReader", | |
| "comments": "", | |
| "type": "org.apache.nifi.csv.CSVReader", | |
| "bundle": { | |
| "group": "org.apache.nifi", | |
| "artifact": "nifi-record-serialization-services-nar", | |
| "version": "2.0.0-SNAPSHOT" | |
| }, | |
| "properties": { | |
| "ignore-csv-header": "false", | |
| "schema-branch": null, | |
| "Trim double quote": "true", | |
| "CSV Format": "custom", | |
| "Quote Character": "\"", | |
| "csvutils-allow-duplicate-header-names": "true", | |
| "Value Separator": ",", | |
| "Record Separator": "\\n", | |
| "Timestamp Format": null, | |
| "Escape Character": "\\", | |
| "Date Format": null, | |
| "Null String": null, | |
| "Skip Header Line": "false", | |
| "Trim Fields": "true", | |
| "schema-name": "${schema.name}", | |
| "schema-registry": null, | |
| "csv-reader-csv-parser": "jackson-csv", | |
| "Time Format": null, | |
| "Comment Marker": null, | |
| "schema-access-strategy": "infer-schema", | |
| "schema-version": null, | |
| "csvutils-character-set": "UTF-8", | |
| "schema-text": "${avro.schema}" | |
| }, | |
| "propertyDescriptors": { | |
| "ignore-csv-header": { | |
| "name": "ignore-csv-header", | |
| "displayName": "Ignore CSV Header Column Names", | |
| "identifiesControllerService": false, | |
| "sensitive": false, | |
| "dynamic": false | |
| }, | |
| "schema-branch": { | |
| "name": "schema-branch", | |
| "displayName": "Schema Branch", | |
| "identifiesControllerService": false, | |
| "sensitive": false, | |
| "dynamic": false | |
| }, | |
| "Trim double quote": { | |
| "name": "Trim double quote", | |
| "displayName": "Trim double quote", | |
| "identifiesControllerService": false, | |
| "sensitive": false, | |
| "dynamic": false | |
| }, | |
| "CSV Format": { | |
| "name": "CSV Format", | |
| "displayName": "CSV Format", | |
| "identifiesControllerService": false, | |
| "sensitive": false, | |
| "dynamic": false | |
| }, | |
| "Quote Character": { | |
| "name": "Quote Character", | |
| "displayName": "Quote Character", | |
| "identifiesControllerService": false, | |
| "sensitive": false, | |
| "dynamic": false | |
| }, | |
| "csvutils-allow-duplicate-header-names": { | |
| "name": "csvutils-allow-duplicate-header-names", | |
| "displayName": "Allow Duplicate Header Names", | |
| "identifiesControllerService": false, | |
| "sensitive": false, | |
| "dynamic": false | |
| }, | |
| "Value Separator": { | |
| "name": "Value Separator", | |
| "displayName": "Value Separator", | |
| "identifiesControllerService": false, | |
| "sensitive": false, | |
| "dynamic": false | |
| }, | |
| "Record Separator": { | |
| "name": "Record Separator", | |
| "displayName": "Record Separator", | |
| "identifiesControllerService": false, | |
| "sensitive": false, | |
| "dynamic": false | |
| }, | |
| "Timestamp Format": { | |
| "name": "Timestamp Format", | |
| "displayName": "Timestamp Format", | |
| "identifiesControllerService": false, | |
| "sensitive": false, | |
| "dynamic": false | |
| }, | |
| "Escape Character": { | |
| "name": "Escape Character", | |
| "displayName": "Escape Character", | |
| "identifiesControllerService": false, | |
| "sensitive": false, | |
| "dynamic": false | |
| }, | |
| "Date Format": { | |
| "name": "Date Format", | |
| "displayName": "Date Format", | |
| "identifiesControllerService": false, | |
| "sensitive": false, | |
| "dynamic": false | |
| }, | |
| "Null String": { | |
| "name": "Null String", | |
| "displayName": "Null String", | |
| "identifiesControllerService": false, | |
| "sensitive": false, | |
| "dynamic": false | |
| }, | |
| "Skip Header Line": { | |
| "name": "Skip Header Line", | |
| "displayName": "Treat First Line as Header", | |
| "identifiesControllerService": false, | |
| "sensitive": false, | |
| "dynamic": false | |
| }, | |
| "Trim Fields": { | |
| "name": "Trim Fields", | |
| "displayName": "Trim Fields", | |
| "identifiesControllerService": false, | |
| "sensitive": false, | |
| "dynamic": false | |
| }, | |
| "schema-name": { | |
| "name": "schema-name", | |
| "displayName": "Schema Name", | |
| "identifiesControllerService": false, | |
| "sensitive": false, | |
| "dynamic": false | |
| }, | |
| "schema-registry": { | |
| "name": "schema-registry", | |
| "displayName": "Schema Registry", | |
| "identifiesControllerService": true, | |
| "sensitive": false, | |
| "dynamic": false | |
| }, | |
| "csv-reader-csv-parser": { | |
| "name": "csv-reader-csv-parser", | |
| "displayName": "CSV Parser", | |
| "identifiesControllerService": false, | |
| "sensitive": false, | |
| "dynamic": false | |
| }, | |
| "Time Format": { | |
| "name": "Time Format", | |
| "displayName": "Time Format", | |
| "identifiesControllerService": false, | |
| "sensitive": false, | |
| "dynamic": false | |
| }, | |
| "Comment Marker": { | |
| "name": "Comment Marker", | |
| "displayName": "Comment Marker", | |
| "identifiesControllerService": false, | |
| "sensitive": false, | |
| "dynamic": false | |
| }, | |
| "schema-access-strategy": { | |
| "name": "schema-access-strategy", | |
| "displayName": "Schema Access Strategy", | |
| "identifiesControllerService": false, | |
| "sensitive": false, | |
| "dynamic": false | |
| }, | |
| "schema-version": { | |
| "name": "schema-version", | |
| "displayName": "Schema Version", | |
| "identifiesControllerService": false, | |
| "sensitive": false, | |
| "dynamic": false | |
| }, | |
| "csvutils-character-set": { | |
| "name": "csvutils-character-set", | |
| "displayName": "Character Set", | |
| "identifiesControllerService": false, | |
| "sensitive": false, | |
| "dynamic": false | |
| }, | |
| "schema-text": { | |
| "name": "schema-text", | |
| "displayName": "Schema Text", | |
| "identifiesControllerService": false, | |
| "sensitive": false, | |
| "dynamic": false | |
| } | |
| }, | |
| "controllerServiceApis": [{ | |
| "type": "org.apache.nifi.serialization.RecordReaderFactory", | |
| "bundle": { | |
| "group": "org.apache.nifi", | |
| "artifact": "nifi-standard-services-api-nar", | |
| "version": "2.0.0-SNAPSHOT" | |
| } | |
| }], | |
| "scheduledState": "DISABLED", | |
| "bulletinLevel": "WARN", | |
| "componentType": "CONTROLLER_SERVICE", | |
| "groupIdentifier": "a226434e-78d6-3846-bb1f-55680679a3b3" | |
| }], | |
| "variables": {}, | |
| "defaultFlowFileExpiration": "0 sec", | |
| "defaultBackPressureObjectThreshold": 10000, | |
| "defaultBackPressureDataSizeThreshold": "1 GB", | |
| "scheduledState": "ENABLED", | |
| "executionEngine": "INHERITED", | |
| "maxConcurrentTasks": 1, | |
| "statelessFlowTimeout": "1 min", | |
| "componentType": "PROCESS_GROUP", | |
| "flowFileConcurrency": "UNBOUNDED", | |
| "flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE" | |
| }, | |
| "externalControllerServices": {}, | |
| "parameterContexts": {}, | |
| "flowEncodingVersion": "1.0", | |
| "parameterProviders": {}, | |
| "latest": false | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment