Skip to content

Instantly share code, notes, and snippets.

@RouNNdeL
Created February 9, 2019 11:57
Show Gist options
  • Save RouNNdeL/184faf107f454579d0929a3745716d16 to your computer and use it in GitHub Desktop.
Save RouNNdeL/184faf107f454579d0929a3745716d16 to your computer and use it in GitHub Desktop.
Node.js file that updates my IP Geolocation list using https://ipstack.com/ API
/*
* MIT License
*
* Copyright (c) 2019 Krzysztof "RouNdeL" Zdulski
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KI"ND",EXPRESS OR
* IMPLI"ED",INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILI"TY", * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLA"IM",DAMAGES OR OTHER
* LIABILI"TY",WHETHER IN AN ACTION OF CONTRA"CT",TORT OR OTHERWI"SE",ARISING FR"OM", * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
const request = require("request");
const mysql = require("mysql");
const regions = require("./regions");
const IP_REGEX = /^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$/;
const IP_PRIVATE_REGEX = /(^127\.)|(^10\.)|(^172\.1[6-9]\.)|(^172\.2[0-9]\.)|(^172\.3[0-1]\.)|(^192\.168\.)/;
const conn = mysql.createConnection({
host: process.env.DB_HOST,
user: process.env.DB_USER,
password: process.env.DB_PASS,
database: process.env.DB_DATABASE
});
function getM49ByCountry(country_code) {
for(let continent in regions) {
if(regions.hasOwnProperty(continent)) {
for(let sub in regions[continent].sub) {
if(regions[continent].sub.hasOwnProperty(sub)) {
if(regions[continent].sub[sub].countries.indexOf(country_code) >= 0) {
return {continent: continent, sub: sub};
}
}
}
}
}
return null;
}
function loadIp(ip) {
let url = `http://api.ipstack.com/${ip}?hostname=1&access_key=${process.env.IPSTACK_API_KEY}`;
return new Promise((resolve, reject) => {
request(url, function(error, response, body) {
if(error === null) {
resolve(JSON.parse(body));
} else {
reject(error);
}
});
})
}
async function loadIps(ips) {
const data = [];
for(let i = 0; i < ips.length; i++) {
data.push(await loadIp(ips[i]));
}
return data;
}
function filterIps(ips) {
const a = [];
for(let i = 0; i < ips.length; i++) {
if(ips[i].match(IP_REGEX) !== null && ips[i].match(IP_PRIVATE_REGEX) === null) {
a.push(ips[i]);
}
}
return a;
}
function fetchMissingIps() {
const sql = `SELECT apache_http.remote_host FROM apache_http LEFT JOIN ip_geolocation
ON ip_geolocation.ip = apache_http.remote_host WHERE ip_geolocation.ip IS NULL GROUP BY remote_host`;
return new Promise((resolve, reject) => {
conn.query(sql, function(error, results, fields) {
if(error !== null) {
reject(error);
}
const ips = [];
for(let i = 0; i < results.length; i++) {
ips.push(results[i].remote_host);
}
resolve(filterIps(ips));
});
});
}
function insertGeoData(data_array) {
const insert_data = [];
for(let i = 0; i < data_array.length; i++) {
const row = [];
const {continent, sub} = getM49ByCountry(data_array[i].country_code);
row.push(data_array[i].ip);
row.push(data_array[i].hostname);
row.push(continent);
row.push(regions[continent].name);
row.push(sub);
row.push(regions[continent].sub[sub].name);
row.push(data_array[i].country_code);
row.push(data_array[i].country_name);
row.push(data_array[i].region_code !== null ? `${data_array[i].country_code}-${data_array[i].region_code}` : null);
row.push(data_array[i].region_name);
row.push(data_array[i].city);
row.push(data_array[i].zip);
row.push(`${data_array[i].latitude},${data_array[i].longitude}`);
insert_data.push(row);
}
const sql = "INSERT INTO ip_geolocation (ip, hostname, continent_code, continent_name, subcontinent_code, subcontinent_name, country_code, country_name, region_code, region_name, city, zip, location) VALUES ?"
return new Promise((resolve, reject) => {
conn.query(sql, [insert_data], function(error) {
if(error !== null)
reject(error);
else
resolve();
});
});
}
async function run() {
conn.connect();
console.log("Connected to DB");
const ips = await fetchMissingIps();
console.log(`Got ${ips.length} missing ips: `, ips);
if(ips.length > 0) {
const geo_data = await loadIps(ips);
console.log("Got IP geo data");
await insertGeoData(geo_data);
console.log("Finished insert");
}
conn.end();
}
run();
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment