Skip to content

Instantly share code, notes, and snippets.

@ziitmy
Created May 2, 2020 19:57
Show Gist options
  • Save ziitmy/06721822dc082a48546d7cd369eaf642 to your computer and use it in GitHub Desktop.
Save ziitmy/06721822dc082a48546d7cd369eaf642 to your computer and use it in GitHub Desktop.
This file would essentially read data from a csv file obtained from kaggle admin data and would give the required admin data output
var admin = require("firebase-admin");
var geohash = require('ngeohash');
var simplify = require('simplify-js');
const geoarea = require('geo-area')
const fetch = require("node-fetch");
const fs = require("fs");
const csv = require("csv-parser");
const geolib = require("geolib");
results = []
geohashIndividualArray = []
var serviceAccount2 = require("./administrative-divisions.json");
var second = admin.initializeApp({
credential: admin.credential.cert(serviceAccount2),
databaseURL: "https://administrative-divisions.firebaseio.com"
}, "second");
const dbAdministrative = second.firestore();
const readCSV = (csvfile) => {
return new Promise((resolve, reject) => {
fs.createReadStream(csvfile)
.pipe(csv())
.on('data', (row) => {
results.push(row)
})
.on('end', () => {
console.log('CSV file successfully processed')
resolve(results)
})
});
}
//Gets all the polygon coordinates of the bounding region of any admin place
const getLatLongData = (combinedKey) => {
return new Promise(async(resolve, reject) => {
await fetch(`https://nominatim.openstreetmap.org/search.php?q=${combinedKey}&polygon_geojson=1&format=json`)
.then((response) => {
resolve(response.json())
})
.catch((error) => {
reject(error);
})
});
}
//Gets the polygon coordinates data and stores it in a format as a GeoPoint as required in the database
const convertForCloud = (afterSimplify) => {
forCloud = [];
afterSimplify.forEach(e => {
try {
forCloud.push(new admin.firestore.GeoPoint(e.x, e.y));
} catch (error) {
console.log('Error in converting the geohash');
}
})
return forCloud;
}
//Gets all polygon coordinates data and stores it in a format as a GeoPoint as required in the database
const convertAllPolygonCoordinatesForCloud = (AllPolygonCoordinates) => {
AllPolygonCoordinatesForCloud = [];
AllPolygonCoordinates.forEach(e => {
try {
AllPolygonCoordinatesForCloud.push(new admin.firestore.GeoPoint(e[1], e[0]));
} catch (error) {
console.log('Error in converting the polygon coordinates');
}
})
return AllPolygonCoordinatesForCloud;
}
//Converts polygon coordinates to geohash all of defined precision
const getGeohashArray = (forCloud) => {
geohashArray = [];
forCloud.forEach((e) => {
geohashh = geohash.encode(e._latitude, e._longitude, precision=8);
// console.log(geohashh);
geohashArray.push(geohashh);
})
return geohashArray;
}
//Gets all the geohashes available for the from a shorter length to the maximum length
const getAllGeoHashesArray = (geohashArray, a, b) => {
allgeoHashes = [];
for(var i=0; i< geohashArray.length; i++) {
for(var j=a; j<= b; j++) {
allgeoHashes.push(geohashArray[i].substring(0, j)) ;
}
}
allUniqueGeohashes = allgeoHashes.filter((a, b) => allgeoHashes.indexOf(a) === b)
return allUniqueGeohashes;
}
deleteCollection = (db, CollectionName) => {
return new Promise(async (resolve, reject) => {
if(CollectionName != null || CollectionName != '') {
console.log("Deleting the data for: ", CollectionName)
var querySnapshot = await db.collection(CollectionName).get() ;
for (doc of querySnapshot.docs) {
await db.collection(CollectionName).doc(doc.id).delete() ;
};
resolve("Deleted successfully");
}
else {
reject("Deletion unsuccessful");
}
});
}
//function to get city wise corona data
async function trials(row){
console.log("->",row.asciiname)
await getLatLongData(row.asciiname)
.then(async (data)=>{
console.log('The control is inside then now');
if(data[0].geojson.type === "Point") {
console.log("The control is inside unavailable")
console.log("data unavailable")
let data2 = {
"hasData": false,
"geoHashes": [],
"all_polygon_coordinates":[],
"simplified_polygon_coordinates": [],
"location": data[0].display_name,
"boundingbox":data[0].boundingbox,
}
let setDoc1 = await dbAdministrative.collection('admin-places-test').doc(data[0].display_name).set(data2);
} else if(data[0].geojson.type === "Polygon" || data[0].geojson.type === "MultiPolygon") {
console.log("The control is inside available")
console.log(data[0].geojson.coordinates[0].length,"line 82");
console.log("for-> ", row.asciiname)
if(data[0].class != "boundary") {
if(data[0].geojson.coordinates[0].length<15) {
var arrayCoordinates = data[0].geojson.coordinates[0][0];
} else {
var arrayCoordinates = data[0].geojson.coordinates[0];
}
} else {
if(data[1].geojson.coordinates[0].length<15) {
var arrayCoordinates = data[1].geojson.coordinates[0][0];
} else {
var arrayCoordinates = data[1].geojson.coordinates[0];
}
}
// console.log(arrayCoordinates);
forSimplify = [];
arrayCoordinates.forEach(e => {
// console.log(e);
forSimplify.push({x:e[1], y:e[0]});
})
//Getting area of the given latitude and longitude coordinates
var area = geolib.getAreaOfPolygon( arrayCoordinates );
var areaInKm2 = geolib.convertArea(area, 'km2');
console.log("Area",area);
//Getting center of the given latitude and longitude coordinates
var center = geolib.getCenter( arrayCoordinates );
// console.log(forSimplify);
console.log(forSimplify.length,"line 90");
//Function used to simplify and reduce the number of coordinates
afterSimplify = simplify(forSimplify, 0.001, false);;
// console.log(afterSimplify,"line 92");
console.log("After simplify length:",afterSimplify.length,"line 93");
//To convert the coordinates in firebase format
arrayCoordinates = convertAllPolygonCoordinatesForCloud(arrayCoordinates);
//To convert the simplified coordinates in firebase format
forCloud = convertForCloud(afterSimplify);
console.log("For cloud length",forCloud.length)
geohashArray = getGeohashArray(forCloud);
console.log("Geohash array length->>>",geohashArray.length);
console.log("Each geohash length", geohashArray[0].length);
allUniqueGeohashes = getAllGeoHashesArray(geohashArray, 1, 8);
console.log(allUniqueGeohashes.length);
//To get all geohashes of respective lengths
for(var i=0; i<8; i++) {
geohashIndividualArray[i] = getAllGeoHashesArray(geohashArray, i, i+1);
}
//To remove the improper length geohashes and to keep only the correct length in each array
for(var i=1; i<=7; i++) {
eachGeohashArray = geohashIndividualArray[i];
for( var j = 0; j < eachGeohashArray.length; j++){
if (eachGeohashArray[j].length != (i+1)) {
eachGeohashArray.splice(j, 1);
}
}
geohashIndividualArray[i] = eachGeohashArray;
}
//To remove the first index since it is null
geohashIndividualArray[0] = geohashIndividualArray[0].splice(1, 1);
let data2 = {
"hasData": true,
"all_polygon_coordinates":arrayCoordinates,
"simplified_polygon_coordinates": forCloud,
"location": data[0].display_name,
"area":areaInKm2,
"geohash_1": geohashIndividualArray[0],
"geohash_2": geohashIndividualArray[1],
"geohash_3": geohashIndividualArray[2],
"geohash_4": geohashIndividualArray[3],
"geohash_5": geohashIndividualArray[4],
"geohash_6": geohashIndividualArray[5],
"geohash_7": geohashIndividualArray[6],
"geohash_8": geohashIndividualArray[7],
"center":center
}
let setDoc1 = await dbAdministrative.collection('admin-places-test').doc(data[0].display_name).set(data2);
}
})
.catch((error)=>{
console.log(error);
});
}
adminDataFunction = async () => {
console.log('Entered adminData functions')
readCSV('./split_data/output_1.csv')
.then( async (results) => {
try {
for(i = 0; i < 1000; i++) {
row = results[i];
console.log(row);
trials(row);
}
} catch(e) {
console.log(e);
}
});
}
adminDataFunction();
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment