Transform Raw JSON Fasterize log to Apache Combined Log Format

fasterizeLogsToApacheLogFormat.js

/**
  * Convert JSON fasterize format to Apache Log Format
  *
  {
    "client_ip": "23.254.250.19",
    "content_length": 25736,
    "content_type": "text/html;charset=UTF-8",
    "domain": "www.clarins.fr",
    "fstrz_flags": "o,sc,c",
    "method": "GET",
    "protocol": "http",
    "referer": "http://www.clarins.fr/corps/300/?utm_term=Baume%20Corps%20Super%20Hydratant%20%20clarins&gclid=CMHkpovwzdUCFS2zswod-8YCPw&utm_campaign=Bing_FR_Brand_Generic&gclsrc=ds&utm_content=Body_Cat%C3%A9gorie_Hydration&utm_source=google",
    "response_time": 0.001,
    "status": "200",
    "timestamp": "2017-08-11T00:00:55.937Z",
    "url": "/eau-dynamisante-grand-format---recharge/0064310.html",
    "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:41.0) Gecko/20100101 Firefox/41.0",
    "user_agent_class": "desktop_recent"
  }

  to
    LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"" combined

  */
const fs = require('fs');
const moment = require('moment');

var rawLogsFile = fs.readFileSync(process.argv[2]);

var logsArray = JSON.parse(rawLogsFile);

logsArray.forEach((logObj) => {
  const clientIP = Array.isArray(logObj['client_ip']) ? logObj['client_ip'][0] : logObj['client_ip'];
  const date = moment(logObj['timestamp']);
  const dateString = date.format('DD/MMM/YYYY:HH:mm:ss ZZ');

  console.log(`${logObj['domain']} ${clientIP} -  - [${dateString}] "${logObj['method']} ${logObj['url']} ${logObj['protocol'].toUpperCase()}" ${logObj['status']} ${logObj['content_length']} "${logObj['referer']}" "${logObj['user_agent']}"`);
});

FasterizeLogToApacheFormat.md

Transform Raw JSON Fasterize log to Apache Combined Log Format

This script use node and the package moment.js. This is useful for Screaming Frog.

Installation

npm install moment

Usage

node fasterizeLogsToApacheLogFormat.js jsonFile.log > apache.log

Example

[
{
  "client_ip": "23.254.250.19",
  "content_length": 25736,
  "content_type": "text/html;charset=UTF-8",
  "domain": "www.clarins.fr",
  "fstrz_flags": "o,sc,c",
  "method": "GET",
  "protocol": "http",
  "referer": "http://www.clarins.fr/corps/300/?utm_term=Baume%20Corps%20Super%20Hydratant%20%20clarins&gclid=CMHkpovwzdUCFS2zswod-8YCPw&utm_campaign=Bing_FR_Brand_Generic&gclsrc=ds&utm_content=Body_Cat%C3%A9gorie_Hydration&utm_source=google",
  "response_time": 0.001,
  "status": "200",
  "timestamp": "2017-08-11T00:00:55.937Z",
  "url": "/eau-dynamisante-grand-format---recharge/0064310.html",
  "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:41.0) Gecko/20100101 Firefox/41.0",
  "user_agent_class": "desktop_recent"
}
]

Output

www.clarins.fr 66.249.76.147 -  - [12/Aug/2017:02:00:01 +0200] "GET /on/demandware.static/-/Library-Sites-clarins-v3/default/dw4e22bb5d/content/joli-rouge-brillant-2016/bottom-page/img/pic-texture11.jpg?frz-v145 HTTPS" 200 2740 "" "Googlebot-Image/1.0"