Skip to content

Instantly share code, notes, and snippets.

@alistairhenderson
Last active August 1, 2023 09:02
Show Gist options
  • Save alistairhenderson/40236705d062cfe6e005e1df22392594 to your computer and use it in GitHub Desktop.
Save alistairhenderson/40236705d062cfe6e005e1df22392594 to your computer and use it in GitHub Desktop.
This will create a Node-Red flow. The information for this can be seen at https://open-source.blog/2023/07/21/getting-node-red-ready-for-aws-s3-access/
[
{
"id": "2dd7c4b1fdb409ab",
"type": "tab",
"label": "Sync Files to S3",
"disabled": false,
"info": "# USE\r\nThis syncs all of the Live_data files every 30 minutes to the S3 bucket",
"env": []
},
{
"id": "60027a0a1146f7aa",
"type": "inject",
"z": "2dd7c4b1fdb409ab",
"name": "",
"props": [
{
"p": "payload"
},
{
"p": "topic",
"vt": "str"
}
],
"repeat": "",
"crontab": "0 0-22 * * *",
"once": false,
"onceDelay": 0.1,
"topic": "",
"payload": "",
"payloadType": "date",
"x": 210,
"y": 220,
"wires": [
[
"73abdd77c90a9047"
]
]
},
{
"id": "c7e98b9c8f3e6859",
"type": "file in",
"z": "2dd7c4b1fdb409ab",
"name": "Read the files",
"filename": "filepath",
"filenameType": "msg",
"format": "utf8",
"chunk": false,
"sendError": false,
"encoding": "base64",
"allProps": false,
"x": 540,
"y": 560,
"wires": [
[
"22c8298da7b2d9a4"
]
]
},
{
"id": "747312053e2cc3d3",
"type": "readdir",
"z": "2dd7c4b1fdb409ab",
"name": "Loop Through copy_data directory",
"dir": "/opt/systems/data/live_data/",
"as": "multi",
"recursive": true,
"outproperty": "filepath",
"x": 280,
"y": 560,
"wires": [
[
"c7e98b9c8f3e6859"
]
]
},
{
"id": "5048f651534218ed",
"type": "debug",
"z": "2dd7c4b1fdb409ab",
"name": "File Listing",
"active": false,
"tosidebar": true,
"console": false,
"tostatus": false,
"complete": "true",
"targetType": "full",
"statusVal": "",
"statusType": "auto",
"x": 1490,
"y": 420,
"wires": []
},
{
"id": "22c8298da7b2d9a4",
"type": "function",
"z": "2dd7c4b1fdb409ab",
"name": "Create all of the paramaters",
"func": "// Get the full path from the payload\nconst fullPath = msg.filepath;\n\n// Extract the filename from the full path\nconst filename = fullPath.substring(fullPath.lastIndexOf('/') + 1);\n\n// Remove the filename from the full path to get the folder path\nconst folderPath = fullPath.substring(0, fullPath.lastIndexOf('/'));\n\n// Extract the foldername from the folder path\nconst foldername = folderPath.substring(folderPath.lastIndexOf('/') + 1);\n\n// Extract the filetype from the filename\nconst filetype = filename.substring(filename.lastIndexOf('.') + 1);\n\n\nconst lineno = `line_no_${msg.line_no.padStart(2, '0')}`;\n\n// Concatenate msg.line_no with foldername\n//const resultString = `${lineno}/${foldername}`;\nconst fullFoldername = `${lineno}/${foldername}`;\n\n\n// Create an object with the extracted values\nconst result = {\n foldername: fullFoldername,\n filename: filename.replace(`.${filetype}`, ''),\n filetype: filetype,\n};\n\n// Output the result in msg.payload\nmsg.fileparamaters = result;\n\n// Return the modified message\nreturn msg;",
"outputs": 1,
"noerr": 0,
"initialize": "",
"finalize": "",
"libs": [],
"x": 860,
"y": 560,
"wires": [
[
"660ee98a0bc2acae"
]
]
},
{
"id": "660ee98a0bc2acae",
"type": "change",
"z": "2dd7c4b1fdb409ab",
"name": "Create Elements for the S3 Write",
"rules": [
{
"t": "move",
"p": "payload",
"pt": "msg",
"to": "base64",
"tot": "msg"
},
{
"t": "set",
"p": "folderName",
"pt": "msg",
"to": "fileparamaters.foldername",
"tot": "msg"
},
{
"t": "set",
"p": "fileType",
"pt": "msg",
"to": "fileparamaters.filetype",
"tot": "msg"
},
{
"t": "move",
"p": "filename",
"pt": "msg",
"to": "file_path",
"tot": "msg"
},
{
"t": "set",
"p": "userId",
"pt": "msg",
"to": "fileparamaters.filename",
"tot": "msg"
}
],
"action": "",
"property": "",
"from": "",
"to": "",
"reg": false,
"x": 1220,
"y": 560,
"wires": [
[
"5048f651534218ed",
"63e4aa2e98c912ec",
"81a988e7dd6f42f9"
]
]
},
{
"id": "73abdd77c90a9047",
"type": "function",
"z": "2dd7c4b1fdb409ab",
"name": "Environmental Variables",
"func": "msg.line_no = '01'\nreturn msg;",
"outputs": 1,
"noerr": 0,
"initialize": "",
"finalize": "",
"libs": [],
"x": 590,
"y": 220,
"wires": [
[
"747312053e2cc3d3"
]
]
},
{
"id": "63e4aa2e98c912ec",
"type": "s3-upload",
"z": "2dd7c4b1fdb409ab",
"name": "write-to-s3",
"target": "payload",
"accessKeyId": "IAM_ID",
"accessKeyIdType": "str",
"secretAccessKey": "IAM-ACCESS-KEY",
"secretAccessKeyType": "str",
"awsRegion": "eu-west-2",
"awsRegionType": "str",
"s3Bucket": "S3-BUCKET-NAME",
"s3BucketType": "str",
"x": 1510,
"y": 560,
"wires": [
[],
[]
]
},
{
"id": "81a988e7dd6f42f9",
"type": "flogger",
"z": "2dd7c4b1fdb409ab",
"name": "Log",
"logfile": "linesync.log",
"inputchoice": "object",
"inputobject": "file_path",
"inputobjectType": "msg",
"inputmoustache": "Recieved payload {{payload}} and topic {{topic}}",
"loglevel": "INFO",
"logconfig": "1fda2671412d7083",
"sendpane": "",
"x": 1490,
"y": 740,
"wires": [
[]
]
},
{
"id": "1fda2671412d7083",
"type": "config-log",
"logname": "FileLogging",
"logdir": "/u10/logging",
"stamp": "utc",
"logstyle": "plain",
"logrotate": true,
"logcompress": true,
"logrotatecount": "1",
"logsize": "1000"
}
]
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment