Skip to content

Instantly share code, notes, and snippets.

@tim-minter

tim-minter/README.md

Forked from dceejay/README.md
Last active April 11, 2017 16:37
Show Gist options
  • Save tim-minter/74b95b0366aeaaf658c19468cda5fcea to your computer and use it in GitHub Desktop.
Save tim-minter/74b95b0366aeaaf658c19468cda5fcea to your computer and use it in GitHub Desktop.
Download ZIP
Dynamic Word Cloud
Raw
README.md

Forked from dceejay's twitter version. Converted to use IBM Connections instead of Twitter. "Intelligent" stop list added. Changed to use the on-line D3 source as opposed to a local copy (needs v3 to work. v4 does not work with this code).

This flow generatates a dynamic Word Cloud type graphic. It takes the output of an IBM Connections Search node as input, and uses d3 to generate a Word Cloud based on the relative frequency of words found.

It includes a stop list to exclude common words and exclude the search term and words in the search term your are searching on.

As well as the flow you also need to download and save this web page

https://gist.githubusercontent.com/dceejay/1d9b0959cf7cc36f9f8c/raw/3bb9263537d996031d2337fd64582a323a5e9168/cloud.html

and the d3 word cloud library from here

https://github.com/jasondavies/d3-cloud

You just need the d3.layout.cloud.js (and the license :-)

Place these 2 (or 3) files into the node-red/public directory then fire up the flow below. Then browse to http://localhost:1880/cloud.html

Raw
cloud.html
<!DOCTYPE html>
<meta charset="utf-8">
<title>Node-RED Wordcloud</title>
<body align=middle>
<script src="https://d3js.org/d3.v3.min.js"></script>
<script src="./d3.layout.cloud.js"></script>
<script>
function wordCloud(selector) {
var fill = d3.scale.category20b();
var svg = d3.select(selector).append("svg")
.attr("width", 1024)
.attr("height", 768)
.append("g")
.attr("transform", "translate(512,384)");
function draw(words) {
//Use the 'text' attribute (the word itself) to identity unique elements.
var cloud = svg.selectAll("g text")
.data(words, function(d) { return d.text; })
//Entering words
cloud.enter()
.append("text")
.style("font-family", "Impact")
.style("fill", function(d, i) { return fill(i); })
.attr("text-anchor", "middle")
.attr('font-size', 2)
.text(function(d) { return d.text; });
//Entering and existing words
cloud.transition()
.duration(600)
.style("font-size", function(d) { return d.size + "px"; })
.attr("transform", function(d) {
return "translate(" + [d.x, d.y] + ")rotate(" + d.rotate + ")";
})
.style("fill-opacity", 1);
//Exiting words
cloud.exit()
.transition()
.duration(200)
.style('fill-opacity', 1e-6)
.attr('font-size', 1)
.remove();
}
return {
//Recompute the word cloud for a new set of words. This method will
// asynchronously call draw when the layout has been computed.
update: function(words) {
d3.layout.cloud().size([1024, 768])
.words(words)
.padding(2)
.rotate(function() { return ~~(Math.random() * -2) * 90; })
//.rotate(function(d) { return ~~(Math.random() * 5) * 30 - 60; })
.font("Impact")
.fontSize(function(d) { return d.size; })
.on("end", draw)
.start();
}
}
}
var myWordCloud = wordCloud('body');
var ws;
var server = window.location.hostname;
var wsUri = "ws://"+server+":1880/ws/wordcloud";
function start(wsUri) { // Create the websocket
ws = new WebSocket(wsUri);
ws.onopen = function(evt) {
console.log("CONNECTED");
//document.getElementById("foot").innerHTML = "<font color='#494'>"+ibmfoot+"</font>";
//ws.send("Open for mapping");
};
ws.onclose = function(evt) {
console.log("DISCONNECTED");
//document.getElementById("foot").innerHTML = "<font color='#944'>"+ibmfoot+"</font>";
setTimeout(function(){ start(wsUri) }, 3000); // try to reconnect every 3 secs... bit fast ?
}
// This expects a websocket message with data as a stringified object containing at least name, lat and lon
ws.onmessage = function (evt) {
//console.log("DATA",evt.data);
var wds = JSON.parse(evt.data);
keysSorted = Object.keys(wds).sort(function(a,b){return wds[a]-wds[b]});
//console.log(keysSorted);
var wrds=[];
for (var key in wds) {
wrds.push({ text:key, size:10 + 20 * wds[key] });
}
myWordCloud.update(wrds);
}
ws.onerror = function(evt) {
console.log("ERROR",evt);
//document.getElementById("foot").innerHTML = "<font color='#f00'>"+ibmfoot+"</font>";
}
}
start(wsUri);
</script>
</body>
Raw
flow.json
[{"id":"d7ff7cf5.975e48","type":"function","z":"16feff23.748181","name":"Count words in array","func":"var index = {};\n\n//var toIgnore = [ \"and\", \"for\", \"but\", \"the\", \"&lt\", \"&gt\", \"his\", \"her\", \"pre\", \"are\", \"&amp\", \"with\"];\n\nvar toIgnore = flow.get('stopWords')||\"\";\n\n\nconsole.log(toIgnore);\n\nfunction countWords(sentence) {\n\twords = sentence\n\t\t.replace(/[.,?!;()\"'-]/g, \" \")\n\t\t.replace(/\\s+/g, \" \")\n\t\t//.toLowerCase()\n\t\t.split(\" \");\n\t\n\twords.forEach(function (word) {\n\t\tif ((word.length > 2)&&( isNaN(Number(word)) )&&( toIgnore.indexOf(word.toLowerCase()) == -1 )) {\n\t\t if (!(index.hasOwnProperty(word))) {\n\t\t index[word] = 0;\n\t\t }\n\t\t index[word]++;\n\t\t}\n\t});\n\t\n\n}\n\nfor (var i in msg.payload) {\n\tcountWords(msg.payload[i]);\n}\n\nmsg.payload = JSON.stringify(index);\nmsg.stopWords = toIgnore;\nreturn msg;","outputs":1,"noerr":0,"x":1476.0000343322754,"y":699.666576385498,"wires":[["4c875e90.27911","a5cd3214.ab9688","ad7f768f.758cc"]]},{"id":"4c875e90.27911","type":"websocket out","z":"16feff23.748181","name":"","server":"acf0a873.d12ed","x":1774.999912261963,"y":698.3332281112671,"wires":[]},{"id":"c375b720.e42f8","type":"inject","z":"16feff23.748181","name":"","topic":"","payload":"\"tim minter\"","payloadType":"str","repeat":"","crontab":"","once":false,"x":246.66668701171875,"y":378.3332796096802,"wires":[["1df07a5c.61e33e","694379cd.423428"]]},{"id":"1df07a5c.61e33e","type":"function","z":"16feff23.748181","name":"Translate msg.payload to msg.query","func":"msg.query = msg.payload;\nreturn msg;","outputs":1,"noerr":0,"x":338.50001525878906,"y":456.99993228912354,"wires":[["8a924470.987ef"]]},{"id":"8a924470.987ef","type":"SimpleSearch","z":"16feff23.748181","name":"Search All Status Updates","server":"","query":"","theScope":"status_updates","sortKey":"date","sortOrder":"desc","maxResults":"50","limitCB":true,"sinceDate":"07/08/2016","sinceCB":false,"untilDate":"06/04/2017","untilCB":false,"mytags":"","x":648.5,"y":610.3332328796387,"wires":[["818ece04.a37978"]]},{"id":"818ece04.a37978","type":"splitter","z":"16feff23.748181","name":"","property":"payload","x":859.2500228881836,"y":610.4165840148926,"wires":[["1848d520.bc02fb"]]},{"id":"1848d520.bc02fb","type":"function","z":"16feff23.748181","name":"Get Title","func":"msg.payload = msg.payload.title;\nreturn msg;","outputs":1,"noerr":0,"x":869.0000343322754,"y":699.9166975021362,"wires":[["5f57099c.6f5e3"]]},{"id":"a5cd3214.ab9688","type":"debug","z":"16feff23.748181","name":"count","active":true,"console":"false","complete":"payload","x":1735.8332862854004,"y":766.6665849685669,"wires":[]},{"id":"3710da36.e222ce","type":"function","z":"16feff23.748181","name":"Build Array","func":"\nvar entries = context.entries || [\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\"];\n\n\tvar foo = msg.payload;\n\tif (foo.indexOf(\"http\") != -1) {\n\t\tfoo = (foo.split(\"http\"))[0];\n\t}\n\t\n\tif (foo.toLowerCase().indexOf(\"job\") != -1) return null;\n\tfoo = foo.replace(/[^\\x00-\\x7F]/g, \"\");\n\t\n\tif (entries.indexOf(foo) == -1) {\n\t\tentries.shift();\n\t\tentries.push(foo);\n\t\tcontext.entries = entries;\n\t\treturn {payload:entries};\n\t}\n\t\nreturn null;\n","outputs":1,"noerr":0,"x":1236.6665229797363,"y":699.999927520752,"wires":[["d7ff7cf5.975e48"]]},{"id":"5f57099c.6f5e3","type":"badwords","z":"16feff23.748181","name":"","x":1044.1666641235352,"y":699.9999465942383,"wires":[["3710da36.e222ce"]]},{"id":"694379cd.423428","type":"function","z":"16feff23.748181","name":"stopWords (with search term added)","func":"\nvar stopWords = [\n\"a\",\n\"about\",\n\"above\",\n\"after\",\n\"again\",\n\"against\",\n\"all\",\n\"am\",\n\"an\",\n\"and\",\n\"any\",\n\"are\",\n\"aren't\",\n\"as\",\n\"at\",\n\"be\",\n\"because\",\n\"been\",\n\"before\",\n\"being\",\n\"below\",\n\"between\",\n\"both\",\n\"but\",\n\"by\",\n\"can't\",\n\"cannot\",\n\"could\",\n\"couldn't\",\n\"did\",\n\"didn't\",\n\"do\",\n\"does\",\n\"doesn't\",\n\"doing\",\n\"don't\",\n\"down\",\n\"during\",\n\"each\",\n\"few\",\n\"for\",\n\"from\",\n\"further\",\n\"had\",\n\"hadn't\",\n\"has\",\n\"hasn't\",\n\"have\",\n\"haven't\",\n\"having\",\n\"he\",\n\"he'd\",\n\"he'll\",\n\"he's\",\n\"her\",\n\"here\",\n\"here's\",\n\"hers\",\n\"herself\",\n\"him\",\n\"himself\",\n\"his\",\n\"how\",\n\"how's\",\n\"i\",\n\"i'd\",\n\"i'll\",\n\"i'm\",\n\"i've\",\n\"if\",\n\"in\",\n\"into\",\n\"is\",\n\"isn't\",\n\"it\",\n\"it's\",\n\"its\",\n\"itself\",\n\"let's\",\n\"me\",\n\"more\",\n\"most\",\n\"mustn't\",\n\"my\",\n\"myself\",\n\"no\",\n\"nor\",\n\"not\",\n\"of\",\n\"off\",\n\"on\",\n\"once\",\n\"only\",\n\"or\",\n\"other\",\n\"ought\",\n\"our\",\n\"ours\",\n\"ourselves\",\n\"out\",\n\"over\",\n\"own\",\n\"same\",\n\"shan't\",\n\"she\",\n\"she'd\",\n\"she'll\",\n\"she's\",\n\"should\",\n\"shouldn't\",\n\"so\",\n\"some\",\n\"such\",\n\"than\",\n\"that\",\n\"that's\",\n\"the\",\n\"their\",\n\"theirs\",\n\"them\",\n\"themselves\",\n\"then\",\n\"there\",\n\"there's\",\n\"these\",\n\"they\",\n\"they'd\",\n\"they'll\",\n\"they're\",\n\"they've\",\n\"this\",\n\"those\",\n\"through\",\n\"to\",\n\"too\",\n\"under\",\n\"until\",\n\"up\",\n\"very\",\n\"was\",\n\"wasn't\",\n\"we\",\n\"we'd\",\n\"we'll\",\n\"we're\",\n\"we've\",\n\"were\",\n\"weren't\",\n\"what\",\n\"what's\",\n\"when\",\n\"when's\",\n\"where\",\n\"where's\",\n\"which\",\n\"while\",\n\"who\",\n\"who's\",\n\"whom\",\n\"why\",\n\"why's\",\n\"with\",\n\"won't\",\n\"would\",\n\"wouldn't\",\n\"you\",\n\"you'd\",\n\"you'll\",\n\"you're\",\n\"you've\",\n\"your\",\n\"yours\",\n\"yourself\",\n\"yourselves\"];\n\n\nvar searchTerm = msg.payload;\nvar cleanedSearchTerm = searchTerm.replace(/[\"']/g, \"\");\nstopWords.push(cleanedSearchTerm);\n\nvar splitSearchTerm = cleanedSearchTerm.split(\" \");\nsplitSearchTerm.forEach(function(item, index, array) {\n stopWords.push(item);\n});\n\nconsole.log(stopWords);\nflow.set('stopWords',stopWords);\n ","outputs":1,"noerr":0,"x":694.1666641235352,"y":378.33331871032715,"wires":[[]]},{"id":"ad7f768f.758cc","type":"debug","z":"16feff23.748181","name":"","active":true,"console":"false","complete":"stopWords","x":1764.166648864746,"y":626.6666507720947,"wires":[]},{"id":"6e00a3b7.210b2c","type":"comment","z":"16feff23.748181","name":"The search term and the individual words in the search term are added to the stop list","info":"","x":840.8333587646484,"y":333.33333587646484,"wires":[]},{"id":"1a3334cc.5d3773","type":"comment","z":"16feff23.748181","name":"If nothing is found, the error \"Missing <ENTRY> element\" is returned","info":"","x":772.5000152587891,"y":558.3333530426025,"wires":[]},{"id":"5b680203.e8d60c","type":"comment","z":"16feff23.748181","name":"Edit the stop list here","info":"","x":640.8333206176758,"y":288.3333339691162,"wires":[]},{"id":"acf0a873.d12ed","type":"websocket-listener","path":"/ws/wordcloud","wholemsg":"false"}]
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment