Created
February 9, 2019 11:57
-
-
Save RouNNdeL/184faf107f454579d0929a3745716d16 to your computer and use it in GitHub Desktop.
Node.js file that updates my IP Geolocation list using https://ipstack.com/ API
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
/* | |
* MIT License | |
* | |
* Copyright (c) 2019 Krzysztof "RouNdeL" Zdulski | |
* | |
* Permission is hereby granted, free of charge, to any person obtaining a copy | |
* of this software and associated documentation files (the "Software"), to deal | |
* in the Software without restriction, including without limitation the rights | |
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell | |
* copies of the Software, and to permit persons to whom the Software is | |
* furnished to do so, subject to the following conditions: | |
* | |
* The above copyright notice and this permission notice shall be included in all | |
* copies or substantial portions of the Software. | |
* | |
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KI"ND",EXPRESS OR | |
* IMPLI"ED",INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILI"TY", * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE | |
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLA"IM",DAMAGES OR OTHER | |
* LIABILI"TY",WHETHER IN AN ACTION OF CONTRA"CT",TORT OR OTHERWI"SE",ARISING FR"OM", * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
* SOFTWARE. | |
*/ | |
const request = require("request"); | |
const mysql = require("mysql"); | |
const regions = require("./regions"); | |
const IP_REGEX = /^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$/; | |
const IP_PRIVATE_REGEX = /(^127\.)|(^10\.)|(^172\.1[6-9]\.)|(^172\.2[0-9]\.)|(^172\.3[0-1]\.)|(^192\.168\.)/; | |
const conn = mysql.createConnection({ | |
host: process.env.DB_HOST, | |
user: process.env.DB_USER, | |
password: process.env.DB_PASS, | |
database: process.env.DB_DATABASE | |
}); | |
function getM49ByCountry(country_code) { | |
for(let continent in regions) { | |
if(regions.hasOwnProperty(continent)) { | |
for(let sub in regions[continent].sub) { | |
if(regions[continent].sub.hasOwnProperty(sub)) { | |
if(regions[continent].sub[sub].countries.indexOf(country_code) >= 0) { | |
return {continent: continent, sub: sub}; | |
} | |
} | |
} | |
} | |
} | |
return null; | |
} | |
function loadIp(ip) { | |
let url = `http://api.ipstack.com/${ip}?hostname=1&access_key=${process.env.IPSTACK_API_KEY}`; | |
return new Promise((resolve, reject) => { | |
request(url, function(error, response, body) { | |
if(error === null) { | |
resolve(JSON.parse(body)); | |
} else { | |
reject(error); | |
} | |
}); | |
}) | |
} | |
async function loadIps(ips) { | |
const data = []; | |
for(let i = 0; i < ips.length; i++) { | |
data.push(await loadIp(ips[i])); | |
} | |
return data; | |
} | |
function filterIps(ips) { | |
const a = []; | |
for(let i = 0; i < ips.length; i++) { | |
if(ips[i].match(IP_REGEX) !== null && ips[i].match(IP_PRIVATE_REGEX) === null) { | |
a.push(ips[i]); | |
} | |
} | |
return a; | |
} | |
function fetchMissingIps() { | |
const sql = `SELECT apache_http.remote_host FROM apache_http LEFT JOIN ip_geolocation | |
ON ip_geolocation.ip = apache_http.remote_host WHERE ip_geolocation.ip IS NULL GROUP BY remote_host`; | |
return new Promise((resolve, reject) => { | |
conn.query(sql, function(error, results, fields) { | |
if(error !== null) { | |
reject(error); | |
} | |
const ips = []; | |
for(let i = 0; i < results.length; i++) { | |
ips.push(results[i].remote_host); | |
} | |
resolve(filterIps(ips)); | |
}); | |
}); | |
} | |
function insertGeoData(data_array) { | |
const insert_data = []; | |
for(let i = 0; i < data_array.length; i++) { | |
const row = []; | |
const {continent, sub} = getM49ByCountry(data_array[i].country_code); | |
row.push(data_array[i].ip); | |
row.push(data_array[i].hostname); | |
row.push(continent); | |
row.push(regions[continent].name); | |
row.push(sub); | |
row.push(regions[continent].sub[sub].name); | |
row.push(data_array[i].country_code); | |
row.push(data_array[i].country_name); | |
row.push(data_array[i].region_code !== null ? `${data_array[i].country_code}-${data_array[i].region_code}` : null); | |
row.push(data_array[i].region_name); | |
row.push(data_array[i].city); | |
row.push(data_array[i].zip); | |
row.push(`${data_array[i].latitude},${data_array[i].longitude}`); | |
insert_data.push(row); | |
} | |
const sql = "INSERT INTO ip_geolocation (ip, hostname, continent_code, continent_name, subcontinent_code, subcontinent_name, country_code, country_name, region_code, region_name, city, zip, location) VALUES ?" | |
return new Promise((resolve, reject) => { | |
conn.query(sql, [insert_data], function(error) { | |
if(error !== null) | |
reject(error); | |
else | |
resolve(); | |
}); | |
}); | |
} | |
async function run() { | |
conn.connect(); | |
console.log("Connected to DB"); | |
const ips = await fetchMissingIps(); | |
console.log(`Got ${ips.length} missing ips: `, ips); | |
if(ips.length > 0) { | |
const geo_data = await loadIps(ips); | |
console.log("Got IP geo data"); | |
await insertGeoData(geo_data); | |
console.log("Finished insert"); | |
} | |
conn.end(); | |
} | |
run(); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment