Skip to content

Instantly share code, notes, and snippets.

@dajve
Last active September 8, 2018 14:45
Show Gist options
  • Save dajve/ee24187ae49fbcb37dba087708bcb50f to your computer and use it in GitHub Desktop.
Save dajve/ee24187ae49fbcb37dba087708bcb50f to your computer and use it in GitHub Desktop.
Magento shell script to parse system logs to CSV
<?php
require_once 'abstract.php';
class Djv_Shell_AppLog extends Mage_Shell_Abstract
{
const DEFAULT_SOURCE_FILE = 'system.log';
const LOG_LINE_REGEX = "/^(?P<date>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}([+-]\d{2}:\d{2})?)\b\s*\b(?P<errcode>[A-Z]+)\b\s\((?P<errno>\d)\):\s*(?P<message>.*?)(\s+in\s(?P<file>[^\s]+)\s+on\sline\s(?<line>\d+))?$/";
/**
* Djv_Shell_AppLog constructor.
*/
public function __construct()
{
parent::__construct();
$this->csv = new Varien_File_Csv();
}
/**
* @throws Mage_Core_Exception
*/
public function run()
{
switch (true) {
case in_array('to-csv', $_SERVER['argv']) :
$sourceFile = $this->getArg('source') ?? self::DEFAULT_SOURCE_FILE;
$destFile = $this->getArg('dest');
if (!$destFile) {
$extension = pathinfo($sourceFile, PATHINFO_EXTENSION);
$destFile = substr($sourceFile, 0, 0 - (strlen($extension) + 1)) . '.csv';
}
$this->toCsv($sourceFile, $destFile);
echo PHP_EOL."Done".PHP_EOL;
break;
default :
echo $this->usageHelp();
break;
}
}
/**
* @param string $line
* @return array
*/
public function extractData($line)
{
$line = trim($line);
$return = [
'date' => null,
'errcode' => null,
'errno' => null,
'type' => null,
'subtype' => null,
'message' => null,
'file' => null,
'lineno' => null,
// 'line' => $line,
];
$matches = [];
preg_match(self::LOG_LINE_REGEX, $line, $matches);
if (empty($matches)) {
return $return;
}
$return = array_merge($return, [
'date' => $matches['date'],
'errcode' => $matches['errcode'],
'errno' => $matches['errno'],
'message' => $matches['message'],
'file' => $matches['file'],
'lineno' => $matches['lineno'],
]);
$return = array_merge($return, $this->getErrorClassification($matches['message']));
return $return;
}
/**
* @param string $message
* @return array
*/
public function getErrorClassification($message)
{
$return = [
'type' => null,
'subtype' => null,
];
$matches = [];
switch (true) {
case preg_match("/SQLSTATE\[(\d+)\]/", $message,$matches) :
$return['type'] = 'SQL Error';
$sqlStateCode = $matches[1] ? (int)$matches[1] : null;
switch (true) {
case $sqlStateCode >= 21000 && $sqlStateCode < 22000:
$return['subtype'] = "21xxx : cardinality violation";
break;
case $sqlStateCode >= 22000 && $sqlStateCode < 23000 :
$return['subtype'] = "22xxx : data exception";
break;
case $sqlStateCode >= 23000 && $sqlStateCode < 24000 :
$return['subtype'] = "23xxx : integrity constraint violation";
break;
case $sqlStateCode >= 24000 && $sqlStateCode < 25000 :
$return['subtype'] = "24xxx : invalid cursor state ";
break;
case $sqlStateCode >= 25000 && $sqlStateCode < 26000 :
$return['subtype'] = "25xxx : invalid transaction state ";
break;
default :
$return['subtype'] = $matches[1];
break;
}
break;
case false !== strpos($message, "Not valid template file:") :
$return['type'] = 'Magento error';
$return['subtype'] = 'Invalid template';
break;
case false !== strpos($message, "Notice: Undefined offset") :
case false !== strpos($message, "Notice: Undefined index") :
case false !== strpos($message, "Notice: Undefined variable") :
$return['type'] = 'PHP Notice';
$return['subtype'] = 'Undefined item';
break;
case preg_match('/Missing argument \d+ for/', $message) :
$return['type'] = 'PHP Warning';
$return['subtype'] = 'Missing argument';
break;
case preg_match("/(include|require)(_once)?\([^\)]+\): failed to open stream:/", $message) :
case preg_match("/(include|require)(_once)?\(\): Failed opening/", $message) :
$return['type'] = 'PHP Warning';
$return['subtype'] = 'Missing include';
break;
case 0 === strpos($message, "Notice: ") :
$return['type'] = 'PHP Notice';
break;
case 0 === strpos($message, "Warning: ") :
$return['type'] = 'PHP Warning';
break;
}
return $return;
}
/**
* @param string $sourceFile
* @param string $destFile
* @return bool
* @throws Mage_Core_Exception
*/
public function toCsv($sourceFile, $destFile)
{
$sourceFile = $this->getJailedFilepath($sourceFile);
if (!file_exists($sourceFile)) {
throw new Mage_Core_Exception(__("Cannot find source file %s", $sourceFile));
}
$destFile = $this->getJailedFilepath($destFile);
$addHeaders = true;
if (file_exists($destFile)) {
switch ($this->getArg('destAction')) {
case 'append' :
$addHeaders = false;
break;
case 'overwrite' :
unlink($destFile);
break;
case 'archive' :
default :
$this->archiveFile($destFile);
break;
}
}
$sourceFileObject = new SplFileObject($sourceFile);
$destFileHandle = fopen($destFile, 'a+');
if (!$destFileHandle) {
throw new RuntimeException(sprintf(
"Cannot open '%s' for writing",
$destFile
));
}
$rows = 0;
while ($sourceFileObject->valid()) {
$line = $sourceFileObject->fgets();
$lineData = $this->extractData($line);
if (!array_filter($lineData)) {
continue;
}
if ($addHeaders && 0 === $rows) {
$this->csv->fputcsv($destFileHandle, array_keys($lineData));
}
$this->csv->fputcsv($destFileHandle, $lineData);
$rows++;
}
fclose($destFileHandle);
return true;
}
/**
* @param string $filepath
* @return bool
*/
public function archiveFile($filepath)
{
if (!file_exists($filepath)) {
return false;
}
$io = new Varien_Io_File();
$from = $filepath;
$to = $filepath . '.' . date('YmdHis');
return $io->mv($from, $to);
}
/**
* @param string $filepath
* @return string
*/
public function getJailedFilepath($filepath)
{
return Mage::getBaseDir('log') . DS . preg_replace('/^[\.\/\\\\]+/', null, $filepath);
}
/**
* @return string
*/
public function usageHelp()
{
return <<<USAGE
Usage: php -f djv-applog.php -- [options]
php -f djv-applog.php -- to-csv --source system.log --dest system.csv
to-csv Parses a log file (not exception log at this time) and creates a csv version
If destination exists, it will be archived first
--source <path> File from which to read (jailed to var/log)
--dest <path> File to which transformations are saved (jailed to var/log)
--destAction <archive*,overwrite,append>
USAGE;
}
}
$shell = new Djv_Shell_AppLog();
$shell->run();
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment