Skip to content

Instantly share code, notes, and snippets.

@Syfaro
Last active January 1, 2016 20:59
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save Syfaro/8201026 to your computer and use it in GitHub Desktop.
Save Syfaro/8201026 to your computer and use it in GitHub Desktop.
DNS resolution collector thing, I use RAW (https://github.com/densitydesign/raw) for stats
<?php
// this script outputs a CSV representation of the data with the domain name, the number of times the domain name has been visited, the first time it was visited (by hour) and the last time it was visited (by hour)
$conn = new PDO('mysql:host=;dbname=', '', '');
$items = $conn->query('SELECT `logs`.`id`, `logs`.`t`, `domains`.`domain` FROM `logs` INNER JOIN `domains` ON `logs`.`domain_id` = `domains`.`id`');
$things = array();
foreach($items as $item) {
$things[] = $item;
}
$unique = array();
foreach($things as $thing) {
preg_match('([A-Za-z0-9]+\.[A-Za-z]+$)', $thing['domain'], $matches);
if(!isset($matches[0])) {
continue;
}
$domain = $matches[0];
if(array_key_exists($domain, $unique)) {
$unique[$domain]['count']++;
if($unique[$domain]['first'] > $thing['t']) {
$unique[$domain]['first'] = $thing['t'];
}
if($unique[$domain]['last'] < $thing['t']) {
$unique[$domain]['last'] = $thing['t'];
}
} else {
$unique[$domain] = array();
$unique[$domain]['count'] = 1;
$unique[$domain]['first'] = $thing['t'];
$unique[$domain]['last'] = $thing['t'];
}
}
arsort($unique);
$out = fopen('php://output', 'w');
$csv_array = array();
foreach($unique as $key => $item) {
fputcsv($out, array($key, $item['count'], date('Y-m-d H', $item['first']), date('Y-m-d H', $item['last'])));
}
fclose($out);
// this script is what actually collects the DNS information. you have to run it and set your computer to use ::1 / 127.0.0.1 as your DNS server
var jack = require('dnsjack').createServer();
var mysql = require('mysql');
var pool = mysql.createPool({
host: '',
user: '',
password: '',
database: ''
});
pool.getConnection(function(err, connection) {
if(err)
return console.error(err);
connection.query('SET SESSION innodb_autoinc_lock_mode=0', function(err) {
connection.release();
});
});
var handleDomain = function(domain, connection, callback) {
connection.query('SELECT `id` FROM `domains` WHERE `domain` = ?', [domain], function(err, rows) {
if(err) {
console.error(err);
return callback(err);
}
if(rows[0] === undefined) {
connection.query('INSERT IGNORE INTO `domains` (`domain`) VALUES (?)', [domain], function(err) {
if(err) {
console.error(err);
return callback(err);
}
connection.query('SELECT `id` FROM `domains` WHERE `domain` = ?', [domain], function(err, rows) {
if(err) {
console.error(err);
return callback(err);
}
return callback(err, rows[0].id);
});
});
} else {
return callback(err, rows[0].id);
}
});
};
var addLogEntry = function(domain_id, connection, callback) {
try {
connection.query('INSERT INTO `logs` (t, domain_id) VALUES (?, ?)', [Math.round((new Date()).getTime() / 1000), domain_id], function(err) {
if(err) {
console.error(err);
return callback(err);
}
callback(err);
});
} catch (e) {
console.error(e);
}
};
jack.on('resolve', function(domain) {
console.log('Resolving %s', domain);
pool.getConnection(function(err, connection) {
try {
handleDomain(domain, connection, function(err, id) {
if(err) {
connection.release();
return console.error(err);
}
addLogEntry(id, connection, function(err) {
if(err) {
connection.release();
return console.error(err);
}
connection.release();
});
});
} catch (e) {
connection.release();
return console.error(e);
}
});
});
jack.listen();
SET SQL_MODE = "NO_AUTO_VALUE_ON_ZERO";
SET time_zone = "+00:00";
CREATE TABLE IF NOT EXISTS `domains` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`domain` varchar(767) NOT NULL,
PRIMARY KEY (`id`),
UNIQUE KEY `domain` (`domain`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1 ;
CREATE TABLE IF NOT EXISTS `logs` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`t` text,
`domain_id` int(11) NOT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1 ;
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment