Skip to content

Instantly share code, notes, and snippets.

@vgw-rhysc

vgw-rhysc/Cf_LogParser.js

Last active December 22, 2017 09:11
Show Gist options
  • Save vgw-rhysc/eaf2972418f4ecb1c7541023b81897ae to your computer and use it in GitHub Desktop.
Save vgw-rhysc/eaf2972418f4ecb1c7541023b81897ae to your computer and use it in GitHub Desktop.
Download ZIP
Clickstream samples with Snowplow
Raw
Cf_LogParser.js
const CloudFrontParser = require('cloudfront-log-parser');
const fs = require('fs');
const zlib = require('zlib');
var gzippedFilePt = "/Users/rhys/Downloads/E25TC780L4OA1V.2017-12-22-05.e1c5f2ed.gz";
var processAccessEntry = function(access){
var queryString = access['cs-uri-query'];
var decodedQueryString = decodeURI(decodeURI(queryString));
var arr = decodedQueryString.split("&")
let dict = {}
arr.forEach(element => {
let kvp = element.split("=");
dict[kvp[0]] = kvp[1]
});
if(dict['cx']){
var buf = Buffer.from(dict['cx'], 'base64');
dict['cx_decoded'] = buf.toString('utf8');
}
console.log(access);
console.log(dict);
}
const parser = new CloudFrontParser({ format: 'web' });
parser.on('readable', function () {
let access;
while (access = parser.read()) {
processAccessEntry(access)
}
});
fs.createReadStream(gzippedFilePt).pipe(zlib.createGunzip()).pipe(parser);
Raw
index.html
<html>
<head>
<script
src="https://code.jquery.com/jquery-3.2.1.min.js"
integrity="sha256-hwg4gsxgFZhOsEEamdOYGBf13FyQuiTwlAQgxVSNgt4="
crossorigin="anonymous"></script>
<!-- Snowplow starts plowing -->
<script type="text/javascript">
;(function(p,l,o,w,i,n,g){if(!p[i]){p.GlobalSnowplowNamespace=p.GlobalSnowplowNamespace||[];
p.GlobalSnowplowNamespace.push(i);p[i]=function(){(p[i].q=p[i].q||[]).push(arguments)
};p[i].q=p[i].q||[];n=l.createElement(o);g=l.getElementsByTagName(o)[0];n.async=1;
n.src=w;g.parentNode.insertBefore(n,g)}}(window,document,"script","//d1fc8wv8zag5ca.cloudfront.net/2.8.2/sp.js","snowplow"));
window.snowplow('newTracker', 'cf', 'd15z3c5ji1th4u.cloudfront.net', { // Initialise a tracker
appId: 'my-awesome-app',
//cookieDomain: '{{MY-COOKIE-DOMAIN}}'
});
//https://github.com/snowplow/snowplow/wiki/2-Specific-event-tracking-with-the-Javascript-tracker
// Turn on page pings every 10 seconds
window.snowplow('enableActivityTracking', 10, 10);
window.snowplow(
'trackPageView',
// no custom title
null,
// The usual array of static contexts
[{
schema: 'iglu:com.acme/my_static_context/jsonschema/1-0-0',
data: {
tournament: "Mega Poker tournament",
userid: "rhystestuser123",
}
}],
// Function which returns an array of custom contexts
// Gets called once per page view / page ping
function() {
return [{
schema: 'iglu:com.acme/dynamic_context/jsonschema/1-0-0',
data: {
dynamicValue: new Date().toString()
}
}];
}
);
</script>
<!-- Snowplow stops plowing -->
</head>
<body>
<div>
<button id="btnTrackClick">Track this std event</button>
<button id="btnTrackCustomEventClick">Track this custom event</button>
</div>
<script type="text/javascript">
//snowplow_name_here('trackStructEvent', 'category', 'action', 'label', 'property', 'value');
let trackEvent = function(action, label, prop, value){
window.snowplow('trackStructEvent', 'RhysCTest', action, label, prop, value);
}
let trackCustomEvent = function(){
//This is a sample event from the docs, but use what ever you want. The docs do state you should regsiter your schemas in iglu
window.snowplow('trackSelfDescribingEvent', {
schema: 'iglu:com.acme_company/viewed_product/jsonschema/2-0-0',
data: {
productId: 'ASO01043',
category: 'Dresses',
brand: 'ACME',
returning: true,
price: 49.95,
sizes: ['xs', 's', 'l', 'xl', 'xxl'],
availableSince: new Date(2013,3,7)
}
});
}
document.getElementById("btnTrackClick").onclick = function(){ trackEvent()};
document.getElementById("btnTrackCustomEventClick").onclick = function(){ trackCustomEvent()};
</script>
</body>
</html>
Raw
SampleNodeApp.js
const express = require('express');
const app = express();
app.use(express.static('public'))
app.use('/', express.static('./public/index.html')); //IE the index.html is in a public folder
//start the web server
app.listen(3001, () => {
console.log('We are running - go to http://localhost:3001/');
});
Raw
SampleOutput.txt
{ date: '2017-12-22',
time: '05:59:52',
'x-edge-location': 'MEL50',
'sc-bytes': '479',
'c-ip': '203.63.220.31',
'cs-method': 'GET',
'cs-host': 'd15z3c5ji1th4u.cloudfront.net',
'cs-uri-stem': '/i',
'sc-status': '200',
'cs-referer': 'http://localhost:3001/',
'cs-user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36',
'cs-uri-query': 'stm=1513922392581&e=pp&url=http%3A%2F%2Flocalhost%3A3001%2F&pp_mix=0&pp_max=0&pp_miy=0&pp_may=0&tv=js-2.8.2&tna=cf&aid=my-awesome-app&p=web&tz=Australia%2FPerth&lang=en-GB&cs=UTF-8&f_pdf=1&f_qt=0&f_realp=0&f_wma=0&f_dir=0&f_fla=0&f_java=0&f_gears=0&f_ag=0&res=1920x1080&cd=24&cookie=1&eid=7bc97d35-9af2-4e03-9d69-3a1ad1bb0db8&dtm=1513922392578&vp=765x983&ds=765x983&vid=4&sid=2f3f8650-91b2-4ddf-bd88-7bca99d36161&duid=af3336aa-81a0-42ca-ae2a-3927f5bcc4e3&fp=3793333620&cx=eyJzY2hlbWEiOiJpZ2x1OmNvbS5zbm93cGxvd2FuYWx5dGljcy5zbm93cGxvdy9jb250ZXh0cy9qc29uc2NoZW1hLzEtMC0wIiwiZGF0YSI6W3sic2NoZW1hIjoiaWdsdTpjb20uYWNtZS9teV9zdGF0aWNfY29udGV4dC9qc29uc2NoZW1hLzEtMC0wIiwiZGF0YSI6eyJ0b3VybmFtZW50IjoiTWVnYSBQb2tlciB0b3VybmFtZW50IiwidXNlcmlkIjoicmh5c3Rlc3R1c2VyMTIzIn19LHsic2NoZW1hIjoiaWdsdTpjb20uYWNtZS9keW5hbWljX2NvbnRleHQvanNvbnNjaGVtYS8xLTAtMCIsImRhdGEiOnsiZHluYW1pY1ZhbHVlIjoiRnJpIERlYyAyMiAyMDE3IDEzOjU5OjUyIEdNVCswODAwIChBV1NUKSJ9fV19',
'cs-cookie': '-',
'x-edge-result-type': 'Hit',
'x-edge-request-id': 'SO7YyMd_-_NULVqjbrz34gl8Ies3k6lYWHidmA2SqJzEyttKAWQ63g==',
'x-host-header': 'd15z3c5ji1th4u.cloudfront.net',
'cs-protocol': 'http',
'cs-bytes': '1349',
'time-taken': '0.001',
'x-forwarded-for': '-',
'ssl-protocol': '-',
'ssl-cipher': '-',
'x-edge-response-result-type': 'Hit',
'cs-protocol-version': 'HTTP/1.1' }
{ stm: '1513922392581',
e: 'pp',
url: 'http%3A%2F%2Flocalhost%3A3001%2F',
pp_mix: '0',
pp_max: '0',
pp_miy: '0',
pp_may: '0',
tv: 'js-2.8.2',
tna: 'cf',
aid: 'my-awesome-app',
p: 'web',
tz: 'Australia%2FPerth',
lang: 'en-GB',
cs: 'UTF-8',
f_pdf: '1',
f_qt: '0',
f_realp: '0',
f_wma: '0',
f_dir: '0',
f_fla: '0',
f_java: '0',
f_gears: '0',
f_ag: '0',
res: '1920x1080',
cd: '24',
cookie: '1',
eid: '7bc97d35-9af2-4e03-9d69-3a1ad1bb0db8',
dtm: '1513922392578',
vp: '765x983',
ds: '765x983',
vid: '4',
sid: '2f3f8650-91b2-4ddf-bd88-7bca99d36161',
duid: 'af3336aa-81a0-42ca-ae2a-3927f5bcc4e3',
fp: '3793333620',
cx: 'eyJzY2hlbWEiOiJpZ2x1OmNvbS5zbm93cGxvd2FuYWx5dGljcy5zbm93cGxvdy9jb250ZXh0cy9qc29uc2NoZW1hLzEtMC0wIiwiZGF0YSI6W3sic2NoZW1hIjoiaWdsdTpjb20uYWNtZS9teV9zdGF0aWNfY29udGV4dC9qc29uc2NoZW1hLzEtMC0wIiwiZGF0YSI6eyJ0b3VybmFtZW50IjoiTWVnYSBQb2tlciB0b3VybmFtZW50IiwidXNlcmlkIjoicmh5c3Rlc3R1c2VyMTIzIn19LHsic2NoZW1hIjoiaWdsdTpjb20uYWNtZS9keW5hbWljX2NvbnRleHQvanNvbnNjaGVtYS8xLTAtMCIsImRhdGEiOnsiZHluYW1pY1ZhbHVlIjoiRnJpIERlYyAyMiAyMDE3IDEzOjU5OjUyIEdNVCswODAwIChBV1NUKSJ9fV19',
cx_decoded: '{"schema":"iglu:com.snowplowanalytics.snowplow/contexts/jsonschema/1-0-0","data":[{"schema":"iglu:com.acme/my_static_context/jsonschema/1-0-0","data":{"tournament":"Mega Poker tournament","userid":"rhystestuser123"}},{"schema":"iglu:com.acme/dynamic_context/jsonschema/1-0-0","data":{"dynamicValue":"Fri Dec 22 2017 13:59:52 GMT+0800 (AWST)"}}]}' }
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment