MAX Image Caption Generator Demo

README.md

This flow uses the https://developer.ibm.com/exchanges/models/all/max-image-caption-generator/ deep learning model from the Model Asset Exchange to generate captions that describe the content of an input image, and then displays the image together with the caption in the Node-RED Dashboard. Refer to the documentation for information about the returned message from the model.

Install the following four modules to run this example flow:

• node-red-contrib-model-asset-exchange
• node-red-contrib-camerapi
• node-red-dashboard
• node-red-contrib-image-output

If you are interested to learn more, the process of using the Node-RED Node Generator to create a similar node from a deep learning microservice was documented in this blog post.

flow.json

[
    {
        "id": "1486cc6e.bc2bd4",
        "type": "tab",
        "label": "Image Caption Generator",
        "disabled": false,
        "info": "This flow uses the https://developer.ibm.com/exchanges/models/all/max-image-caption-generator/ deep learning model from the Model Asset Exchange to generate captions that describe the content of an input image, and then displays the image together with the caption in the Node-RED Dashboard. Refer to the documentation for information about the returned message from the model.\n\nInstall the following four modules to run this example flow:\n - [node-red-contrib-model-asset-exchange](https://www.npmjs.com/package/node-red-contrib-model-asset-exchange)\n - [node-red-contrib-camerapi](https://www.npmjs.com/package/node-red-contrib-camerapi)\n - [node-red-dashboard](https://www.npmjs.com/package/node-red-dashboard)\n - [node-red-contrib-image-output](https://www.npmjs.com/package/node-red-contrib-image-output)\n\nThe Dashboard GUI is available at http://localhost:1880/ui\n \n> Note: The image-caption-generator node has been pre-configured to use a hosted evaluation instance. We recommend using your own local or cloud instance for non-development use."
    },
    {
        "id": "f38014b1.9549a8",
        "type": "image-caption-generator",
        "z": "1486cc6e.bc2bd4",
        "service": "82a87cae.a2fec",
        "method": "predict",
        "predict_body": "",
        "predict_bodyType": "str",
        "name": "",
        "x": 750,
        "y": 200,
        "wires": [
            [
                "3ed2f47c.573b5c",
                "2289b1b6.89a9ce"
            ]
        ]
    },
    {
        "id": "3ed2f47c.573b5c",
        "type": "debug",
        "z": "1486cc6e.bc2bd4",
        "name": "Debug MAX Output",
        "active": true,
        "tosidebar": true,
        "console": false,
        "tostatus": false,
        "complete": "true",
        "x": 770,
        "y": 140,
        "wires": []
    },
    {
        "id": "c68d2017.f8bfd",
        "type": "ui_button",
        "z": "1486cc6e.bc2bd4",
        "name": "Capture Image Button (Dashboard)",
        "group": "5bddc39c.8a930c",
        "order": 7,
        "width": 0,
        "height": 0,
        "passthru": false,
        "label": "Capture Image",
        "tooltip": "",
        "color": "",
        "bgcolor": "",
        "icon": "",
        "payload": "true",
        "payloadType": "bool",
        "topic": "",
        "x": 200,
        "y": 160,
        "wires": [
            [
                "abb68f0c.3b8"
            ]
        ]
    },
    {
        "id": "2289b1b6.89a9ce",
        "type": "ui_text",
        "z": "1486cc6e.bc2bd4",
        "group": "5bddc39c.8a930c",
        "order": 5,
        "width": "14",
        "height": "2",
        "name": "",
        "label": "Caption:",
        "format": "{{msg.payload}}",
        "layout": "col-center",
        "x": 1080,
        "y": 260,
        "wires": []
    },
    {
        "id": "ed748012.f352f",
        "type": "ui_template",
        "z": "1486cc6e.bc2bd4",
        "group": "5bddc39c.8a930c",
        "name": "Image Display (Dashboard)",
        "order": 4,
        "width": "14",
        "height": "9",
        "format": "<div ng-bind-html='msg.payload'> </div>",
        "storeOutMessages": false,
        "fwdInMessages": true,
        "templateScope": "local",
        "x": 1020,
        "y": 300,
        "wires": [
            []
        ]
    },
    {
        "id": "963ba4b3.f7a9f8",
        "type": "function",
        "z": "1486cc6e.bc2bd4",
        "name": "base64 -> image element",
        "func": "const b64 = msg.payload\nconst imgSrc = \"data:image/png;base64,\" + b64.toString('base64')\n\nconst element = \"<img src='\"+ imgSrc + \"' />\"\nmsg.payload = element\n\nreturn msg;",
        "outputs": 1,
        "noerr": 0,
        "x": 750,
        "y": 260,
        "wires": [
            [
                "ed748012.f352f"
            ]
        ]
    },
    {
        "id": "79a0b406.65b98c",
        "type": "ui_template",
        "z": "1486cc6e.bc2bd4",
        "group": "5bddc39c.8a930c",
        "name": "CSS Styles (Dashboard)",
        "order": 1,
        "width": 0,
        "height": 0,
        "format": "<style>\nimg {\n  max-width: 85%;\n  padding: 0;\n  margin: auto;\n  display: block;\n}\n</style>",
        "storeOutMessages": true,
        "fwdInMessages": true,
        "templateScope": "global",
        "x": 1030,
        "y": 400,
        "wires": [
            []
        ]
    },
    {
        "id": "57e5b771.a5ba68",
        "type": "comment",
        "z": "1486cc6e.bc2bd4",
        "name": "This flow captures an image using the Raspberry Pi Camera and sends it to a MAX Model, then displays the output on the NodeRED Dashboard.",
        "info": "",
        "x": 580,
        "y": 60,
        "wires": []
    },
    {
        "id": "abb68f0c.3b8",
        "type": "camerapi-takephoto",
        "z": "1486cc6e.bc2bd4",
        "filemode": "0",
        "filename": "",
        "filedefpath": "1",
        "filepath": "",
        "fileformat": "jpeg",
        "resolution": "2",
        "rotation": "0",
        "fliph": "0",
        "flipv": "0",
        "brightness": "50",
        "contrast": "0",
        "sharpness": "0",
        "quality": "80",
        "imageeffect": "none",
        "exposuremode": "auto",
        "iso": "0",
        "agcwait": "1.0",
        "led": "1",
        "awb": "auto",
        "name": "Camera Node",
        "x": 460,
        "y": 220,
        "wires": [
            [
                "963ba4b3.f7a9f8",
                "942702d1.54625",
                "f38014b1.9549a8"
            ]
        ]
    },
    {
        "id": "942702d1.54625",
        "type": "image",
        "z": "1486cc6e.bc2bd4",
        "name": "Image Capture Output",
        "width": 200,
        "x": 580,
        "y": 380,
        "wires": []
    },
    {
        "id": "2091203e.e0d2f",
        "type": "inject",
        "z": "1486cc6e.bc2bd4",
        "name": "Click to Capture Image",
        "topic": "",
        "payload": "true",
        "payloadType": "bool",
        "repeat": "",
        "crontab": "",
        "once": false,
        "onceDelay": 0.1,
        "x": 200,
        "y": 280,
        "wires": [
            [
                "abb68f0c.3b8"
            ]
        ]
    },
    {
        "id": "82a87cae.a2fec",
        "type": "image-caption-generator-service",
        "z": "",
        "host": "https://max-image-caption-generator.max.us-south.containers.appdomain.cloud",
        "name": "cloud"
    },
    {
        "id": "5bddc39c.8a930c",
        "type": "ui_group",
        "z": "",
        "name": "Image Caption Generator",
        "tab": "f099e653.e7d1a8",
        "disp": true,
        "width": "14",
        "collapse": false
    },
    {
        "id": "f099e653.e7d1a8",
        "type": "ui_tab",
        "z": "",
        "name": "MAX Image Caption Generator",
        "icon": "dashboard",
        "disabled": false,
        "hidden": false
    }
]