Skip to content

Instantly share code, notes, and snippets.

@shantanoo-desai
Last active June 12, 2023 17:50
Show Gist options
  • Save shantanoo-desai/bca38860af1e55d0323bbad9cb89fca1 to your computer and use it in GitHub Desktop.
Save shantanoo-desai/bca38860af1e55d0323bbad9cb89fca1 to your computer and use it in GitHub Desktop.
- hosts: all
gather_facts: false
vars_files:
- defaults/main.yml
- vault.yml
tasks:
- name: Do Not Execute Playbook if No Flows File extra variable is mentioned
fail:
msg: >
Please specify the flows file to be deployed on the fleet using:
ansible-playbook deploy_flow.yml -e "flow=<name>.json" --ask-vault-pass
when: flow is not defined
- name: Checking if Flow File Exists in Directory
stat:
path: "{{flows_directory }}/{{ flow }}"
delegate_to: localhost
register: flow_file
- name: Flow File doesn't exist
fail:
msg: "The Flow File does not exist in Path: {{flows_directory}}"
when: not flow_file.stat.exists
- name: Reading the Flow File
set_fact:
flow_json: "{{ lookup('ansible.builtin.file', flows_directory + '/' + flow) | from_json }}"
when: flow_file.stat.exists
- name: Check for Existence of either InfluxDB, MySQL or MQTT nodes in Flows
set_fact:
nodes_exist: true
when: "flow_json | selectattr('type', 'equalto', node) | list | count > 0"
loop:
- influxdb
- mqtt-broker
- mysql
loop_control:
loop_var: node
- name: Dynamic Credential Insertion for InfluxDB, MySQL, or MQTT Configuration Nodes
# Use set_fact on `flow_json` variable because we want to update the same JSON structure
set_fact:
# Extract the dict and combine the credentials block in it
# and merge it back into the list and update `flow_json` fact
flow_json: "{{ flow_json[0:index]
+ [ node | combine(node_replacements[node.type]) ]
+ flow_json[index|int+1:] }}"
# Loop over the list of dictionaries and determine the index of the dict
# which has `type` value either `influxdb`, `mysql` or `mqtt`
when: (nodes_exist is not undefined) and (nodes_exist is true)
loop: "{{ query('ansible.utils.index_of', flow_json, 'in', matched_nodes) }}"
loop_control:
# Use `idx` as the variable name for the index obtained by loop
loop_var: index
# Observe the matches made on the Terminal as <node.id>, <node.type> for better clarity
label: "{{ node.id ~ ', ' ~ node.type }}"
# Variables needed for the loop
vars:
# call each indexed dictionary in the list as `node`
node: "{{ flow_json[index] }}"
# check if `type` is either `influxdb`, `mysql`, or `mqtt` and store it `matched_nodes` variable
matched_nodes: "{{ flow_json | selectattr('type', 'in', node_replacements.keys()) }}"
# The general Structure of the "credentials" block to be inserted if there are `matched_nodes`
# The username, password credentials will be extracted from Ansible-Vault
node_replacements:
influxdb:
credentials:
username: "{{ influxdb.username }}"
password: "{{ influxdb.password }}"
MySQLdatabase:
credentials:
user: "{{ mysql.username }}"
password: "{{ mysql.password }}"
mqtt-broker:
credentials:
username: "{{ mqtt.username }}"
password: "{{ mqtt.password }}"
- name: Preparing Flow File for Deployment
copy:
dest: "{{ flows_directory + '/.' + flow }}"
content: "{{ flow_json | to_nice_json }}"
delegate_to: localhost
- name: Obtain the Authentication Token for Node-RED Instance
ansible.builtin.uri:
# API: /auth/token HTTP POST
# Headers: Content-Type: application/json
# Body:
# client_id: node-red-admin
# grant_type: password
# scope: *
# username: Node-RED instance username
# password: Node-RED instance username
# Expected Response Code: 200
# Response Body:
# {
# "access_token": "<token>",
# "expires_in": 604800,
# "token_type": "Bearer"
# }
url: "{{ nodered_base_url }}/auth/token"
method: POST
body:
client_id: node-red-admin
grant_type: password
scope: "*"
username: "{{ nodered.username }}"
password: "{{ nodered.password }}"
body_format: form-urlencoded
status_code: 200
# Store the Response in a variable called `login`
register: login
when: flow is defined
- name: Upload Dedicated Flow on Each Node-RED Instance in Fleet
ansible.builtin.uri:
# API: /nodered/flows HTTP POST
# Headers:
# Authorization: Bearer <token>
# Content-Type: application/json
# Node-RED-API-Version: v1
# Node-RED-Deployment-Type: full
# Body:
# <Flow JSON File>
# Expected Response Code: 204
# Response body: none
url: "{{ nodered_base_url }}/flows"
method: POST
headers:
# Obtain the Token and its Type from the `login` variable from previous task
Authorization: "{{ login.json.token_type }} {{ login.json.access_token }}"
Content-Type: application/json
Node-RED-API-Version: v1
Node-RED-Deployment-Type: full
# `flow` is a variable containing the <name>.json of the node-RED flow to be deployed
body: "{{ lookup('ansible.builtin.file', flows_directory + '/.' + flow) }}"
body_format: json
status_code: 204
register: flow_upload_status
when: flow is defined
- name: Revoke the Authentication Token for Node-RED Instance
ansible.builtin.uri:
# API: /nodered/auth/revoke HTTP POST
# Headers:
# Authorization: Bearer <token>
# Content-Type: application/json
# Body:
# {"token": "<token>"}
# Expected Response Code: 200
# Response body: none
url: "{{ nodered_base_url }}/auth/revoke"
method: POST
headers:
# Obtain the Token and its Type from the `login` variable from previous task
Authorization: "{{ login.json.token_type }} {{ login.json.access_token }}"
Content-Type: application/json
body: '{"token": "{{ login.json.access_token }}" }'
body_format: json
status_code: 200
- name: Post Deployment Cleanup
file:
name: "{{ flows_directory + '/.' + flow }}"
state: absent
when: flow_upload_status.status == 204
delegate_to: localhost
@shantanoo-desai
Copy link
Author

InfluxDB Flow with multiple Influxdb in and Influxdb out nodes but same configuration

[
    {
        "id": "3272da57099ccc8b",
        "type": "influxdb out",
        "z": "ac2ce998eb52d6a1",
        "influxdb": "3b82b6ebb677a390",
        "name": "",
        "measurement": "test",
        "precision": "",
        "retentionPolicy": "",
        "database": "",
        "retentionPolicyV18Flux": "",
        "org": "",
        "bucket": "",
        "x": 550,
        "y": 220,
        "wires": []
    },
    {
        "id": "022df80929df552d",
        "type": "function",
        "z": "ac2ce998eb52d6a1",
        "name": "single value",
        "func": "data = msg;\nbirthday = data.payload.Birthday;\nfirstname = data.payload.FirstName;\nlastname = data.payload.LastName;\n\nmsg.topic = `INSERT INTO Person (LastName, FirstName, Birthday) VALUES (\"${lastname}\", \"${firstname}\", \"${birthday}\")`\n\nreturn msg;\n\n",
        "outputs": 1,
        "noerr": 0,
        "initialize": "",
        "finalize": "",
        "libs": [],
        "x": 330,
        "y": 220,
        "wires": [
            [
                "3272da57099ccc8b",
                "0c620320df54d9e9"
            ]
        ]
    },
    {
        "id": "4b50f3cf12d784f7",
        "type": "comment",
        "z": "ac2ce998eb52d6a1",
        "name": "Write into DB",
        "info": "",
        "x": 130,
        "y": 180,
        "wires": []
    },
    {
        "id": "7e580b15a8db4ac2",
        "type": "comment",
        "z": "ac2ce998eb52d6a1",
        "name": "Drop",
        "info": "",
        "x": 110,
        "y": 480,
        "wires": []
    },
    {
        "id": "5d6402f7b51d5fb3",
        "type": "inject",
        "z": "ac2ce998eb52d6a1",
        "name": "",
        "props": [
            {
                "p": "payload"
            },
            {
                "p": "topic",
                "vt": "str"
            }
        ],
        "repeat": "",
        "crontab": "",
        "once": false,
        "onceDelay": "",
        "topic": "",
        "payload": "",
        "payloadType": "date",
        "x": 140,
        "y": 520,
        "wires": [
            [
                "a6ba63c5924ff968"
            ]
        ]
    },
    {
        "id": "a6ba63c5924ff968",
        "type": "function",
        "z": "ac2ce998eb52d6a1",
        "name": "Drop DB",
        "func": "msg.query=\"DROP database test;\";\nreturn msg;",
        "outputs": 1,
        "noerr": 0,
        "initialize": "",
        "finalize": "",
        "libs": [],
        "x": 320,
        "y": 520,
        "wires": [
            [
                "84cb8f6e5865614e"
            ]
        ]
    },
    {
        "id": "84cb8f6e5865614e",
        "type": "influxdb in",
        "z": "ac2ce998eb52d6a1",
        "influxdb": "3b82b6ebb677a390",
        "name": "time query",
        "query": "",
        "rawOutput": false,
        "precision": "",
        "retentionPolicy": "",
        "org": "",
        "x": 530,
        "y": 520,
        "wires": [
            [
                "13b5487229b408df"
            ]
        ]
    },
    {
        "id": "13b5487229b408df",
        "type": "debug",
        "z": "ac2ce998eb52d6a1",
        "name": "Drop Database",
        "active": true,
        "tosidebar": true,
        "console": false,
        "tostatus": false,
        "complete": "payload",
        "targetType": "msg",
        "statusVal": "",
        "statusType": "auto",
        "x": 720,
        "y": 520,
        "wires": []
    },
    {
        "id": "c3559bf5851edc50",
        "type": "inject",
        "z": "ac2ce998eb52d6a1",
        "name": "",
        "props": [
            {
                "p": "payload",
                "v": "",
                "vt": "date"
            },
            {
                "p": "topic",
                "v": "",
                "vt": "string"
            }
        ],
        "repeat": "",
        "crontab": "",
        "once": false,
        "topic": "",
        "payload": "",
        "payloadType": "date",
        "x": 140,
        "y": 400,
        "wires": [
            [
                "94912a933da37aa3"
            ]
        ]
    },
    {
        "id": "94912a933da37aa3",
        "type": "function",
        "z": "ac2ce998eb52d6a1",
        "name": "simple query",
        "func": "msg.query=\"select * from test;\";\nreturn msg;",
        "outputs": 1,
        "noerr": 0,
        "x": 330,
        "y": 400,
        "wires": [
            [
                "4802b98bf91f16fe"
            ]
        ]
    },
    {
        "id": "4802b98bf91f16fe",
        "type": "influxdb in",
        "z": "ac2ce998eb52d6a1",
        "influxdb": "3b82b6ebb677a390",
        "name": "time query",
        "query": "",
        "rawOutput": false,
        "precision": "",
        "retentionPolicy": "",
        "x": 530,
        "y": 400,
        "wires": [
            [
                "0add337fc7a8593a"
            ]
        ]
    },
    {
        "id": "0add337fc7a8593a",
        "type": "debug",
        "z": "ac2ce998eb52d6a1",
        "name": "Query DB",
        "active": true,
        "tosidebar": true,
        "console": false,
        "tostatus": false,
        "complete": "payload",
        "targetType": "msg",
        "statusVal": "",
        "statusType": "auto",
        "x": 760,
        "y": 400,
        "wires": []
    },
    {
        "id": "6e9584601325f77d",
        "type": "comment",
        "z": "ac2ce998eb52d6a1",
        "name": "Query",
        "info": "",
        "x": 110,
        "y": 360,
        "wires": []
    },
    {
        "id": "37341ac01c09cce8",
        "type": "inject",
        "z": "ac2ce998eb52d6a1",
        "name": "",
        "props": [
            {
                "p": "payload"
            },
            {
                "p": "topic",
                "vt": "str"
            }
        ],
        "repeat": "",
        "crontab": "",
        "once": true,
        "onceDelay": "",
        "topic": "",
        "payload": "",
        "payloadType": "date",
        "x": 150,
        "y": 120,
        "wires": [
            [
                "117bd7d39fa81cec"
            ]
        ]
    },
    {
        "id": "117bd7d39fa81cec",
        "type": "function",
        "z": "ac2ce998eb52d6a1",
        "name": "create DB",
        "func": "msg.query=\"CREATE database test;\";\nreturn msg;",
        "outputs": 1,
        "noerr": 0,
        "x": 320,
        "y": 120,
        "wires": [
            [
                "295efc8813559a5e"
            ]
        ]
    },
    {
        "id": "295efc8813559a5e",
        "type": "influxdb in",
        "z": "ac2ce998eb52d6a1",
        "influxdb": "3b82b6ebb677a390",
        "name": "time query",
        "query": "",
        "rawOutput": false,
        "precision": "",
        "retentionPolicy": "",
        "org": "",
        "x": 530,
        "y": 120,
        "wires": [
            [
                "b02d4ed96d549d8f"
            ]
        ]
    },
    {
        "id": "b02d4ed96d549d8f",
        "type": "debug",
        "z": "ac2ce998eb52d6a1",
        "name": "Create DB",
        "active": true,
        "tosidebar": true,
        "console": false,
        "tostatus": false,
        "complete": "payload",
        "targetType": "msg",
        "statusVal": "",
        "statusType": "auto",
        "x": 710,
        "y": 120,
        "wires": []
    },
    {
        "id": "21b86b58ba40f775",
        "type": "comment",
        "z": "ac2ce998eb52d6a1",
        "name": "CREATE",
        "info": "",
        "x": 120,
        "y": 80,
        "wires": []
    },
    {
        "id": "0c620320df54d9e9",
        "type": "debug",
        "z": "ac2ce998eb52d6a1",
        "name": "Write to DB",
        "active": true,
        "tosidebar": true,
        "console": false,
        "tostatus": false,
        "complete": "payload",
        "targetType": "msg",
        "statusVal": "",
        "statusType": "auto",
        "x": 530,
        "y": 180,
        "wires": []
    },
    {
        "id": "0db402bfb37ce456",
        "type": "ui_form",
        "z": "ac2ce998eb52d6a1",
        "name": "Person",
        "label": "",
        "group": "905b1b7c51cfbf14",
        "order": 0,
        "width": 0,
        "height": 0,
        "options": [
            {
                "label": "Last Name",
                "value": "LastName",
                "type": "text",
                "required": true,
                "rows": null
            },
            {
                "label": "First Name",
                "value": "FirstName",
                "type": "text",
                "required": true,
                "rows": null
            },
            {
                "label": "Birthday",
                "value": "Birthday",
                "type": "date",
                "required": true,
                "rows": null
            }
        ],
        "formValue": {
            "LastName": "",
            "FirstName": "",
            "Birthday": ""
        },
        "payload": "",
        "submit": "submit",
        "cancel": "cancel",
        "topic": "topic",
        "topicType": "msg",
        "splitLayout": false,
        "className": "",
        "x": 120,
        "y": 220,
        "wires": [
            [
                "022df80929df552d"
            ]
        ]
    },
    {
        "id": "3b82b6ebb677a390",
        "type": "influxdb",
        "z": "ac2ce998eb52d6a1",
        "hostname": "influxdb",
        "port": "8086",
        "database": "test",
        "name": "test db",
        "usetls": false,
        "tls": "",
        "influxdbVersion": "1.x",
        "url": "",
        "rejectUnauthorized": false,
        "credentials": {
            "username": "tester",
            "password": "tester",
            "token": ""
        }
    },
    {
        "id": "905b1b7c51cfbf14",
        "type": "ui_group",
        "name": "Person",
        "tab": "2b82b49d97413831",
        "order": 1,
        "disp": true,
        "width": "6",
        "collapse": false,
        "className": ""
    },
    {
        "id": "2b82b49d97413831",
        "type": "ui_tab",
        "name": "Person",
        "icon": "dashboard",
        "disabled": false,
        "hidden": false
    }
]

jq command

each influxdb in and influxdb out node has a influxdb key in them, upon querying they all end out having the same hash value that is the id value of the node "type": "influxdb"

cat flows.json | jq '.[].influxdb'
"3b82b6ebb677a390"
null
null
null
null
null
"3b82b6ebb677a390"
null
null
null
"3b82b6ebb677a390"
null
null
null
null
"3b82b6ebb677a390"
null
null
null
null
null
null
null

@shantanoo-desai
Copy link
Author

shantanoo-desai commented Jan 26, 2023

Solution 1

from Kristianheljas on Ansible IRC Chat

kristianheljas 16:23:11
So, it is just looping trough indexes that match the items you find in �matched_nodes and replacing each index using list splicing to inject on into the correct place

- hosts: localhost
  gather_facts: false
  vars:
    node_red_flows: [
      {
        "id": "one",
        "type": "influxdb",
        "hostname": "127.0.0.1",
        "port": "8086",
        "protocol": "http",
        "database": "test",
        "name": "testfluxdb",
        "usetls": false,
        "tls": "",
        "influxdbVersion": "1.x",
        "url": "http://localhost:8086",
        "rejectUnauthorized": true
      },
      {
        "id": "f6f2187d.f17ca8",
        "type": "tab",
        "label": "Flow 1",
        "disabled": false,
        "info": ""
      },
      {
        "id": "two",
        "type": "influxdb",
        "hostname": "127.0.0.1",
        "port": "8086",
        "protocol": "http",
        "database": "test",
        "name": "testfluxdb",
        "usetls": false,
        "tls": "",
        "influxdbVersion": "1.x",
        "url": "http://localhost:8086",
        "rejectUnauthorized": true
      },
      {
        "id": "ae7554e7968c9aa1",
        "type": "influxdb in",
        "z": "f6f2187d.f17ca8",
        "influxdb": "398670264ec6e79f",
        "name": "fluxtestdb",
        "query": "",
        "rawOutput": false,
        "precision": "",
        "retentionPolicy": "",
        "org": "organisation",
        "x": 460,
        "y": 300,
        "wires": [
          [ ]
        ]
      }
    ]
  tasks:

    - set_fact:
        node_red_flows: "{{ node_red_flows[0:item] + [node_red_flows[item] | combine(combine_with)] + node_red_flows[item|int+1:] }}"
      loop: "{{ query('ansible.utils.index_of', node_red_flows, 'in', matched_nodes) }}"
      vars:
        matched_nodes: "{{ node_red_flows | selectattr('type', '==', 'influxdb') }}"
        combine_with:
          credentials:
            username: tester
            password: tester

    - debug:
        msg: "{{ node_red_flows }}"

@shantanoo-desai
Copy link
Author

shantanoo-desai commented Jan 26, 2023

Solution

by kristian heljas

explanation

I added loop_control as well to provide nicer output for the item and more meaningul name to the index (opposed to item)
[4:58:27 PM] Using the label option, you can configure how ok: [localhost] => (item=****THIS*****) looks like in the ansible output
[4:59:12 PM] And �loop_var allows you to change the default variable name �item, which in this case might have been confusing

- hosts: localhost
  gather_facts: false
  vars:
    node_red_flows: [
      {
        "id": "one",
        "type": "mysql",
        "hostname": "127.0.0.1",
        "port": "8086",
        "protocol": "http",
        "database": "test",
        "name": "testfluxdb",
        "usetls": false,
        "tls": "",
        "influxdbVersion": "1.x",
        "url": "http://localhost:8086",
        "rejectUnauthorized": true
      },
      {
        "id": "f6f2187d.f17ca8",
        "type": "tab",
        "label": "Flow 1",
        "disabled": false,
        "info": ""
      },
      {
        "id": "two",
        "type": "influxdb",
        "hostname": "127.0.0.1",
        "port": "8086",
        "protocol": "http",
        "database": "test",
        "name": "testfluxdb",
        "usetls": false,
        "tls": "",
        "influxdbVersion": "1.x",
        "url": "http://localhost:8086",
        "rejectUnauthorized": true
      },
      {
        "id": "ae7554e7968c9aa1",
        "type": "influxdb in",
        "z": "f6f2187d.f17ca8",
        "influxdb": "398670264ec6e79f",
        "name": "fluxtestdb",
        "query": "",
        "rawOutput": false,
        "precision": "",
        "retentionPolicy": "",
        "org": "organisation",
        "x": 460,
        "y": 300,
        "wires": [
          [ ]
        ]
      },
      {
        "id": "three",
        "type": "influxdb",
        "hostname": "127.0.0.1",
        "port": "8086",
        "protocol": "http",
        "database": "test",
        "name": "testfluxdb",
        "usetls": false,
        "tls": "",
        "influxdbVersion": "1.x",
        "url": "http://localhost:8086",
        "rejectUnauthorized": true
      }
    ]
  tasks:

    - set_fact:
        node_red_flows: "{{ node_red_flows[0:index] + [node | combine(node_replacements[node.type])] + node_red_flows[index|int+1:] }}"
      loop: "{{ query('ansible.utils.index_of', node_red_flows, 'in', matched_nodes) }}"
      loop_control:
        loop_var: index
        label: "{{ node.id ~ ', ' ~ node.type }}"
      vars:
        node: "{{ node_red_flows[index] }}"
        matched_nodes: "{{ node_red_flows | selectattr('type', 'in', node_replacements.keys()) }}"
        node_replacements:
          influxdb:
            credentials:
              username: influxdb-tester
              password: influxdb-tester
          mysql:
            credentials:
              username: mysql-tester
              password: mysql-tester

    - debug:
        msg: "{{ node_red_flows }}"

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment