Skip to content

Instantly share code, notes, and snippets.

@seanmtracey

seanmtracey/Extract_and_analyse.json

Created November 20, 2017 10:50
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save seanmtracey/75013cd50ce51cc6300becf6a8c1dd09 to your computer and use it in GitHub Desktop.
Save seanmtracey/75013cd50ce51cc6300becf6a8c1dd09 to your computer and use it in GitHub Desktop.
Download ZIP
A Node-red flow for extracting keyframes from videos and analysing them with Watson Visual Recogntion
Raw
Extract_and_analyse.json
[{"id":"146f4acc.4733c5","type":"function","z":"ceee673f.122fb8","name":"Determine File Path","func":"if (msg.req.files) {\n var files = Object.keys(msg.req.files);\n msg.payload.filePath = msg.req.files[files[0]][0].path; \n}\nreturn msg;","outputs":1,"noerr":0,"x":278.5,"y":452,"wires":[["c4cd6b26.2acfc8"]]},{"id":"3bf669a6.9b88e6","type":"httpInMultipart","z":"ceee673f.122fb8","name":"","url":"/analyse","method":"post","fields":"[ { \"name\": \"video\", \"maxCount\" : 1 } ]","swaggerDoc":"","x":145,"y":387,"wires":[["146f4acc.4733c5"]]},{"id":"e1f10989.eab7c8","type":"visual-recognition-v3","z":"ceee673f.122fb8","name":"Analyse frame","apikey":"50a86834a2d8947a442a1ba81e9dde323f074786","image-feature":"classifyImage","lang":"en","x":628,"y":652,"wires":[["1a442f8b.94618","5f0f5910.f08c08"]]},{"id":"5f0f5910.f08c08","type":"function","z":"ceee673f.122fb8","name":"Make Sense","func":"\nconst jobData = flow.get(msg.analysisUUID);\nconst frameData = {\n time : msg.keyframeTimeoffset,\n result : msg.result\n};\n\njobData.frames.push(frameData);\nflow.set(msg.analysisUUID, jobData);\n\nif(jobData.frames.length === jobData.total){\n \n \n jobData.frames.sort( (a, b) => {\n if(a.time > b.time){\n return 1\n } else {\n return -1;\n }\n });\n \n msg.payload = jobData;\n \n \n return msg;\n\n}\n","outputs":1,"noerr":0,"x":766.5,"y":713,"wires":[["4a91b005.51ccb"]]},{"id":"a9429c5e.b1b59","type":"function","z":"ceee673f.122fb8","name":"Filter","func":"if(!msg.finished){\n \n if(msg.firstFrame === true){\n flow.set(msg.analysisUUID, {\n frames : [],\n total : -1\n });\n return [msg, msg];\n }\n return [null, msg];\n} else {\n const existingData = flow.get(msg.analysisUUID);\n existingData.total = msg.totalFrames;\n flow.set(msg.analysisUUID, existingData);\n}\n","outputs":"2","noerr":0,"x":519.5,"y":579,"wires":[["aac384ef.f4d3e8"],["e1f10989.eab7c8"]]},{"id":"4a91b005.51ccb","type":"debug","z":"ceee673f.122fb8","name":"","active":true,"console":"false","complete":"true","x":959.5,"y":636,"wires":[]},{"id":"887589b5.759de8","type":"template","z":"ceee673f.122fb8","name":"HTML","field":"payload","fieldType":"msg","format":"handlebars","syntax":"mustache","template":"<!DOCTYPE html>\n<html lang=\"en\">\n<head>\n <title>Watson Keyframe Visual Recognition Demo</title>\n <style>{{{payload.styles}}}</style>\n</head>\n<body>\n\n<header></header>\n \n<h1>Watson Keyframe Visual Recognition Demo</h1>\n\n<div id=\"dropzone\">\n <p>Drop a media file</p>\n</div>\n\n<div id=\"describe\"></div>\n\n<div id=\"keyframes\">\n <input type=\"text\" placeholder=\"Filter keyframes\" />\n <div id=\"frames\"></div>\n</div>\n\n<script>\n \n {{{payload.script}}}\n \n</script>\n\n</body>\n</html>","output":"str","x":427.5,"y":273,"wires":[["e545c9c5.1eae48"]]},{"id":"189c82ba.8ae70d","type":"http in","z":"ceee673f.122fb8","name":"","url":"/demo","method":"get","upload":false,"swaggerDoc":"","x":143.5,"y":149,"wires":[["574bc94c.630238"]]},{"id":"e545c9c5.1eae48","type":"http response","z":"ceee673f.122fb8","name":"","statusCode":"","headers":{},"x":508.5,"y":321,"wires":[]},{"id":"98a6819b.d2ebe","type":"websocket out","z":"ceee673f.122fb8","name":"Emit Frames","server":"82e881e.8e4028","client":"","x":996.5,"y":562,"wires":[]},{"id":"1a442f8b.94618","type":"function","z":"ceee673f.122fb8","name":"Format Result","func":"return {\n payload : {\n result : msg.result,\n time : msg.keyframeTimeoffset,\n uuid : msg.analysisUUID,\n image : msg.payload,\n frameNumber : msg.frameNumber\n }\n};","outputs":1,"noerr":0,"x":755.5,"y":566,"wires":[["98a6819b.d2ebe","4a91b005.51ccb"]]},{"id":"4cfaea4f.e63364","type":"http response","z":"ceee673f.122fb8","name":"","statusCode":"","headers":{},"x":733.5,"y":448,"wires":[]},{"id":"aac384ef.f4d3e8","type":"function","z":"ceee673f.122fb8","name":"Is OK","func":"return {\n payload : JSON.stringify({\n \"status\" : \"OK\",\n \"analysisUUID\" : msg.analysisUUID\n }),\n res : msg.res\n}","outputs":1,"noerr":0,"x":618.5,"y":510,"wires":[["4cfaea4f.e63364"]]},{"id":"83845e3b.6e93e","type":"template","z":"ceee673f.122fb8","name":"JavaScript","field":"payload.script","fieldType":"msg","format":"javascript","syntax":"mustache","template":"(function(){\n \n function prevent(event){\n event.preventDefault();\n event.stopPropagation();\n }\n \n // Sourced from https://stackoverflow.com/questions/11089732/display-image-from-blob-using-javascript-and-websockets\n function encode (input) {\n var keyStr = \"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=\";\n var output = \"\";\n var chr1, chr2, chr3, enc1, enc2, enc3, enc4;\n var i = 0;\n \n while (i < input.length) {\n chr1 = input[i++];\n chr2 = i < input.length ? input[i++] : Number.NaN; // Not sure if the index\n chr3 = i < input.length ? input[i++] : Number.NaN; // checks are needed here\n \n enc1 = chr1 >> 2;\n enc2 = ((chr1 & 3) << 4) | (chr2 >> 4);\n enc3 = ((chr2 & 15) << 2) | (chr3 >> 6);\n enc4 = chr3 & 63;\n \n if (isNaN(chr2)) {\n enc3 = enc4 = 64;\n } else if (isNaN(chr3)) {\n enc4 = 64;\n }\n output += keyStr.charAt(enc1) + keyStr.charAt(enc2) +\n keyStr.charAt(enc3) + keyStr.charAt(enc4);\n }\n return output;\n }\n \n let analysisResults = [];\n let classifierTerms = {};\n \n var dropZone = document.querySelector('#dropzone');\n const descriptionBox = document.querySelector('#describe');\n const keyframesHolder = document.querySelector('#keyframes');\n const frames = keyframesHolder.querySelector('#frames');\n const filter = keyframesHolder.querySelector('input[type=\"text\"]');\n \n window.addEventListener(\"dragover\",function(e){\n prevent(e);\n document.body.dataset.dragging = \"true\";\n },false);\n \n window.addEventListener(\"drop\",function(e){\n prevent(e);\n document.body.dataset.dragging = \"false\";\n },false);\n \n window.addEventListener('dragleave', function(e){\n prevent(e);\n document.body.dataset.dragging = \"false\";\n }, false);\n \n descriptionBox.addEventListener('click', function(){\n \n if(this.dataset.time){\n const v = dropZone.querySelector('video');\n v.pause();\n v.currentTime = Number(this.dataset.time);\n }\n \n }, false);\n \n filter.addEventListener('keyup', function(){\n \n console.log('KEY');\n \n clearTimeout(this.debounce);\n \n const element = this;\n \n this.debounce = setTimeout(function(){\n console.log('Bounce!');\n \n if(element.value === \"\"){\n Array.from(frames.querySelectorAll('img')).forEach(img => {\n img.dataset.active = 'true';\n });\n \n } else {\n \n Array.from(frames.querySelectorAll('img')).forEach(img => {\n img.dataset.active = 'false';\n });\n \n let matchingFrames = classifierTerms[element.value.toLowerCase()] || [];\n \n Object.keys(classifierTerms).map(term => {\n if(term.indexOf(element.value.toLowerCase()) > -1){\n return classifierTerms[term];\n }\n }).filter(x => { return !!x; } )\n .forEach(setOfFrames => {\n setOfFrames.forEach(f => matchingFrames.push(f));\n });\n \n matchingFrames.forEach(frame => {\n frame.dataset.active = 'true';\n });\n \n }\n \n \n }, 200);\n \n }, false);\n \n dropZone.addEventListener('drop', function(e){\n \n console.log(e);\n \n analysisResults = [];\n classifierTerms = {};\n \n const file = event.dataTransfer.files[0];;\n const reader = new FileReader();\n \n // Create video element to show video;\n const v = document.createElement('video');\n v.src = window.URL.createObjectURL(file);\n \n this.dataset.dropped = 'true';\n this.querySelector('p').dataset.active = 'false';\n \n v.addEventListener('mouseenter', function(){\n this.controls = 'true';\n }, true);\n \n v.addEventListener('mouseleave', function(){\n this.removeAttribute('controls');\n }, true);\n \n v.addEventListener('timeupdate', function(){\n \n let latestResult = undefined;\n \n analysisResults.forEach(result => {\n if(result.time <= v.currentTime){\n \n if(result.result.images){\n if(result.result.images.length > 0){\n latestResult = result;\n }\n }\n \n }\n })\n \n descriptionBox.dataset.time = latestResult.time;\n descriptionBox.textContent = latestResult.time + ' >>> ' + latestResult.result.images[0].classifiers[0].classes.map(r => {return r.class}).join(' : ');\n \n });\n \n const exisitingVideo = dropZone.querySelector('video');\n if(exisitingVideo){\n dropZone.removeChild(exisitingVideo);\n }\n dropZone.appendChild(v);\n \n descriptionBox.textContent = '';\n frames.innerHTML = '';\n filter.value = '';\n \n // We read the file and call the upload function with the result\n reader.onload = function(e){\n console.log(e);\n console.log(reader.result);\n \n var formData = new FormData();\n \n formData.append('video', file);\n \n fetch('/analyse', {\n method : 'POST',\n body : formData\n })\n .then(res => {\n if(res.ok){\n return res.json();\n } else {\n return res;\n }\n })\n .then(function(response){\n console.log(response);\n \n if(response.analysisUUID){\n \n const analysisUUID = response.analysisUUID;\n \n var ws = new WebSocket('ws://' + window.location.host + '/frames');\n \n ws.addEventListener('open', () => {\n console.log('WebSocket connection to server opened.');\n });\n \n ws.addEventListener('message', (e) => {\n \n const serverResults = JSON.parse(e.data);\n serverResults.time = Number(serverResults.time);\n \n if(serverResults.uuid === analysisUUID){\n console.log(analysisUUID, serverResults);\n \n analysisResults.push(serverResults);\n \n analysisResults = analysisResults.sort( (a, b) => {\n if(a.time < b.time){\n return -1;\n } else {\n return 1;\n }\n });\n \n console.log(analysisResults);\n \n var arrayBuffer = serverResults.image.data;\n var bytes = new Uint8Array(arrayBuffer);\n \n var image = document.createElement('img');\n image.src = 'data:image/jpg;base64,' + encode(bytes);\n image.dataset.time = serverResults.time;\n \n image.setAttribute('title', serverResults.time);\n \n image.addEventListener('click', function(){\n console.log(this.dataset.time);\n const video = dropZone.querySelector('video');\n video.pause();\n video.currentTime = Number(this.dataset.time);\n }, false);\n \n frames.appendChild(image);\n \n if(!serverResults.result.error){\n\n serverResults.result.images[0].classifiers[0].classes.forEach(classifier => {\n \n const term = classifier.class.toLowerCase();\n \n if(!classifierTerms[term]){\n classifierTerms[term] = [];\n }\n \n classifierTerms[term].push(image);\n \n });\n \n }\n\n \n }\n \n \n });\n \n ws.addEventListener('close', function(e){\n console.log('The ws connection closed');\n }, false);\n \n }\n \n })\n .catch(err => {\n console.log(err);\n })\n \n };\n reader.readAsArrayBuffer(file);\n \n }, true);\n \n }());\n \n ","output":"str","x":326.5,"y":228,"wires":[["887589b5.759de8"]]},{"id":"1d69ad7a.7aa9a3","type":"comment","z":"ceee673f.122fb8","name":"Render page for demo","info":"","x":171.5,"y":108,"wires":[]},{"id":"9c30cdb.4bb7d3","type":"comment","z":"ceee673f.122fb8","name":"Handle file uploading / analysis","info":"","x":192.5,"y":347,"wires":[]},{"id":"574bc94c.630238","type":"template","z":"ceee673f.122fb8","name":"Style","field":"payload.styles","fieldType":"msg","format":"css","syntax":"mustache","template":"*[data-active='false']{\n display: none;\n}\n\nhtml, body{\n width: 100%;\n height: 100%;\n padding: 0;\n margin: 0;\n overflow: scroll;\n font-family: sans-serif;\n}\n\nbody{\n padding: 1em;\n padding-top: 50px;\n display: flex;\n flex-direction: column;\n height: auto;\n justify-content: center;\n align-items: center;\n box-sizing: border-box;\n}\n\nh1, h2, h3, h4, h5, h6{\n font-weight: 200;\n}\n\nheader{\n width: 100%;\n height: 50px;\n border-bottom: 1px solid #4178be;\n position: fixed;\n top: 0;\n left: 0;\n background-image: url(http://www.uidownload.com/files/529/205/221/ibm-icon.png);\n background-position: 10px;\n background-size: 60px;\n background-color: black;\n background-repeat: no-repeat;\n}\n\n#dropzone{\n max-width: 640px;\n max-height: 450px;\n background: #4178be;\n display: flex;\n align-items: center;\n justify-content: center;\n border-radius: 5px;\n width: 100%;\n padding: 2em 0;\n}\n\n#dropzone[data-dropped=\"true\"]{\n background: black;\n}\n\n#dropzone video{\n width: 100%;\n}\n\n#dropzone p{\n border: white 1px dashed;\n border-radius: 20px;\n color: white;\n padding: 0.5em 1em;\n background: rgba(0,0,0,0.1);\n}\n\n#describe{\n background: black;\n color: white;\n padding: 1em;\n margin: 1em;\n box-sizing: border-box;\n width: 100%;\n max-width: 850px;\n border-radius: 3px;\n text-align: center;\n line-height: 1.75em;\n}\n\n#keyframes{\n width: 100%;\n display: flex;\n flex-direction: row;\n flex-wrap: wrap;\n height: auto;\n background: rgb(245,245,245);\n min-height: 40px;\n padding: 1em;\n box-sizing: border-box;\n margin-top: 1em;\n border-radius: 3px;\n justify-content: center;\n max-width: 850px;\n}\n\n#keyframes input{\n width: 100%;\n font-size: 1em;\n background: transparent;\n border: 0px solid;\n border-bottom: 1px solid #b5b5b5;\n padding-bottom: 0.5em;\n outline: none;\n margin-bottom: 1em;\n}\n\n#keyframes #frames{\n display: flex;\n flex-direction: row;\n flex-wrap: wrap;\n align-items: center;\n justify-content: center;\n max-height: 220px;\n overflow: scroll;\n}\n\n#keyframes #frames img {\n width: 100px;\n height: 65px;\n margin: 0.5em;\n border-radius: 2px;\n cursor: pointer;\n transition: ease-in-out transform 0.1s;\n}\n\n#keyframes #frames img:hover{\n transform: scale(1.5);\n box-shadow: 0 2px 5px black;\n}","output":"str","x":236.5,"y":185,"wires":[["83845e3b.6e93e"]]},{"id":"69ae97af.395288","type":"debug","z":"ceee673f.122fb8","name":"","active":true,"console":"false","complete":"true","x":515.5,"y":450,"wires":[]},{"id":"c4cd6b26.2acfc8","type":"extract-keyframes","z":"ceee673f.122fb8","name":"","x":359.5,"y":521,"wires":[["69ae97af.395288","a9429c5e.b1b59"]]},{"id":"82e881e.8e4028","type":"websocket-listener","z":"","path":"/frames","wholemsg":"false"}]
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment