Created
May 2, 2020 19:57
-
-
Save ziitmy/06721822dc082a48546d7cd369eaf642 to your computer and use it in GitHub Desktop.
This file would essentially read data from a csv file obtained from kaggle admin data and would give the required admin data output
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
var admin = require("firebase-admin"); | |
var geohash = require('ngeohash'); | |
var simplify = require('simplify-js'); | |
const geoarea = require('geo-area') | |
const fetch = require("node-fetch"); | |
const fs = require("fs"); | |
const csv = require("csv-parser"); | |
const geolib = require("geolib"); | |
results = [] | |
geohashIndividualArray = [] | |
var serviceAccount2 = require("./administrative-divisions.json"); | |
var second = admin.initializeApp({ | |
credential: admin.credential.cert(serviceAccount2), | |
databaseURL: "https://administrative-divisions.firebaseio.com" | |
}, "second"); | |
const dbAdministrative = second.firestore(); | |
const readCSV = (csvfile) => { | |
return new Promise((resolve, reject) => { | |
fs.createReadStream(csvfile) | |
.pipe(csv()) | |
.on('data', (row) => { | |
results.push(row) | |
}) | |
.on('end', () => { | |
console.log('CSV file successfully processed') | |
resolve(results) | |
}) | |
}); | |
} | |
//Gets all the polygon coordinates of the bounding region of any admin place | |
const getLatLongData = (combinedKey) => { | |
return new Promise(async(resolve, reject) => { | |
await fetch(`https://nominatim.openstreetmap.org/search.php?q=${combinedKey}&polygon_geojson=1&format=json`) | |
.then((response) => { | |
resolve(response.json()) | |
}) | |
.catch((error) => { | |
reject(error); | |
}) | |
}); | |
} | |
//Gets the polygon coordinates data and stores it in a format as a GeoPoint as required in the database | |
const convertForCloud = (afterSimplify) => { | |
forCloud = []; | |
afterSimplify.forEach(e => { | |
try { | |
forCloud.push(new admin.firestore.GeoPoint(e.x, e.y)); | |
} catch (error) { | |
console.log('Error in converting the geohash'); | |
} | |
}) | |
return forCloud; | |
} | |
//Gets all polygon coordinates data and stores it in a format as a GeoPoint as required in the database | |
const convertAllPolygonCoordinatesForCloud = (AllPolygonCoordinates) => { | |
AllPolygonCoordinatesForCloud = []; | |
AllPolygonCoordinates.forEach(e => { | |
try { | |
AllPolygonCoordinatesForCloud.push(new admin.firestore.GeoPoint(e[1], e[0])); | |
} catch (error) { | |
console.log('Error in converting the polygon coordinates'); | |
} | |
}) | |
return AllPolygonCoordinatesForCloud; | |
} | |
//Converts polygon coordinates to geohash all of defined precision | |
const getGeohashArray = (forCloud) => { | |
geohashArray = []; | |
forCloud.forEach((e) => { | |
geohashh = geohash.encode(e._latitude, e._longitude, precision=8); | |
// console.log(geohashh); | |
geohashArray.push(geohashh); | |
}) | |
return geohashArray; | |
} | |
//Gets all the geohashes available for the from a shorter length to the maximum length | |
const getAllGeoHashesArray = (geohashArray, a, b) => { | |
allgeoHashes = []; | |
for(var i=0; i< geohashArray.length; i++) { | |
for(var j=a; j<= b; j++) { | |
allgeoHashes.push(geohashArray[i].substring(0, j)) ; | |
} | |
} | |
allUniqueGeohashes = allgeoHashes.filter((a, b) => allgeoHashes.indexOf(a) === b) | |
return allUniqueGeohashes; | |
} | |
deleteCollection = (db, CollectionName) => { | |
return new Promise(async (resolve, reject) => { | |
if(CollectionName != null || CollectionName != '') { | |
console.log("Deleting the data for: ", CollectionName) | |
var querySnapshot = await db.collection(CollectionName).get() ; | |
for (doc of querySnapshot.docs) { | |
await db.collection(CollectionName).doc(doc.id).delete() ; | |
}; | |
resolve("Deleted successfully"); | |
} | |
else { | |
reject("Deletion unsuccessful"); | |
} | |
}); | |
} | |
//function to get city wise corona data | |
async function trials(row){ | |
console.log("->",row.asciiname) | |
await getLatLongData(row.asciiname) | |
.then(async (data)=>{ | |
console.log('The control is inside then now'); | |
if(data[0].geojson.type === "Point") { | |
console.log("The control is inside unavailable") | |
console.log("data unavailable") | |
let data2 = { | |
"hasData": false, | |
"geoHashes": [], | |
"all_polygon_coordinates":[], | |
"simplified_polygon_coordinates": [], | |
"location": data[0].display_name, | |
"boundingbox":data[0].boundingbox, | |
} | |
let setDoc1 = await dbAdministrative.collection('admin-places-test').doc(data[0].display_name).set(data2); | |
} else if(data[0].geojson.type === "Polygon" || data[0].geojson.type === "MultiPolygon") { | |
console.log("The control is inside available") | |
console.log(data[0].geojson.coordinates[0].length,"line 82"); | |
console.log("for-> ", row.asciiname) | |
if(data[0].class != "boundary") { | |
if(data[0].geojson.coordinates[0].length<15) { | |
var arrayCoordinates = data[0].geojson.coordinates[0][0]; | |
} else { | |
var arrayCoordinates = data[0].geojson.coordinates[0]; | |
} | |
} else { | |
if(data[1].geojson.coordinates[0].length<15) { | |
var arrayCoordinates = data[1].geojson.coordinates[0][0]; | |
} else { | |
var arrayCoordinates = data[1].geojson.coordinates[0]; | |
} | |
} | |
// console.log(arrayCoordinates); | |
forSimplify = []; | |
arrayCoordinates.forEach(e => { | |
// console.log(e); | |
forSimplify.push({x:e[1], y:e[0]}); | |
}) | |
//Getting area of the given latitude and longitude coordinates | |
var area = geolib.getAreaOfPolygon( arrayCoordinates ); | |
var areaInKm2 = geolib.convertArea(area, 'km2'); | |
console.log("Area",area); | |
//Getting center of the given latitude and longitude coordinates | |
var center = geolib.getCenter( arrayCoordinates ); | |
// console.log(forSimplify); | |
console.log(forSimplify.length,"line 90"); | |
//Function used to simplify and reduce the number of coordinates | |
afterSimplify = simplify(forSimplify, 0.001, false);; | |
// console.log(afterSimplify,"line 92"); | |
console.log("After simplify length:",afterSimplify.length,"line 93"); | |
//To convert the coordinates in firebase format | |
arrayCoordinates = convertAllPolygonCoordinatesForCloud(arrayCoordinates); | |
//To convert the simplified coordinates in firebase format | |
forCloud = convertForCloud(afterSimplify); | |
console.log("For cloud length",forCloud.length) | |
geohashArray = getGeohashArray(forCloud); | |
console.log("Geohash array length->>>",geohashArray.length); | |
console.log("Each geohash length", geohashArray[0].length); | |
allUniqueGeohashes = getAllGeoHashesArray(geohashArray, 1, 8); | |
console.log(allUniqueGeohashes.length); | |
//To get all geohashes of respective lengths | |
for(var i=0; i<8; i++) { | |
geohashIndividualArray[i] = getAllGeoHashesArray(geohashArray, i, i+1); | |
} | |
//To remove the improper length geohashes and to keep only the correct length in each array | |
for(var i=1; i<=7; i++) { | |
eachGeohashArray = geohashIndividualArray[i]; | |
for( var j = 0; j < eachGeohashArray.length; j++){ | |
if (eachGeohashArray[j].length != (i+1)) { | |
eachGeohashArray.splice(j, 1); | |
} | |
} | |
geohashIndividualArray[i] = eachGeohashArray; | |
} | |
//To remove the first index since it is null | |
geohashIndividualArray[0] = geohashIndividualArray[0].splice(1, 1); | |
let data2 = { | |
"hasData": true, | |
"all_polygon_coordinates":arrayCoordinates, | |
"simplified_polygon_coordinates": forCloud, | |
"location": data[0].display_name, | |
"area":areaInKm2, | |
"geohash_1": geohashIndividualArray[0], | |
"geohash_2": geohashIndividualArray[1], | |
"geohash_3": geohashIndividualArray[2], | |
"geohash_4": geohashIndividualArray[3], | |
"geohash_5": geohashIndividualArray[4], | |
"geohash_6": geohashIndividualArray[5], | |
"geohash_7": geohashIndividualArray[6], | |
"geohash_8": geohashIndividualArray[7], | |
"center":center | |
} | |
let setDoc1 = await dbAdministrative.collection('admin-places-test').doc(data[0].display_name).set(data2); | |
} | |
}) | |
.catch((error)=>{ | |
console.log(error); | |
}); | |
} | |
adminDataFunction = async () => { | |
console.log('Entered adminData functions') | |
readCSV('./split_data/output_1.csv') | |
.then( async (results) => { | |
try { | |
for(i = 0; i < 1000; i++) { | |
row = results[i]; | |
console.log(row); | |
trials(row); | |
} | |
} catch(e) { | |
console.log(e); | |
} | |
}); | |
} | |
adminDataFunction(); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment