Skip to content

Instantly share code, notes, and snippets.

@ntkog
Last active November 19, 2019 15:57
Show Gist options
  • Save ntkog/37f14a572383ef0dfdd481d28d73e5c9 to your computer and use it in GitHub Desktop.
Save ntkog/37f14a572383ef0dfdd481d28d73e5c9 to your computer and use it in GitHub Desktop.
GeoGrep - Para cuando quieres filtrar csv's espacialmente por países sin una geodatabase.
const fs = require("fs");
const {chain} = require("stream-chain");
const {parser} = require("stream-csv-as-json");
const {asObjects} = require("stream-csv-as-json/AsObjects");
const {streamValues} = require("stream-json/streamers/StreamValues");
const PolygonLookup = require('polygon-lookup');
const COUNTRYCODES = require('./data/countries_iso_codes.json');
const COUNTRY = process.argv[5];
function validCountry(countryCode) {
return COUNTRYCODES.includes(countryCode);
}
function getCountryBBDD(countryCode) {
let country = require(`./data/countries/${countryCode}.json`);
return new PolygonLookup(country);
}
function setupPipeline(countryCode) {
let BBDD = getCountryBBDD(countryCode);
return chain([
parser({ separator: ","}),
asObjects(),
streamValues(),
data => {
let lat = parseFloat(data.value[process.argv[3]]);
let lon = parseFloat(data.value[process.argv[4]]);
let found = BBDD.search(lon,lat);
return found ? data : null;
},
data => console.log(Object.values(data.value).join(","))
])
}
if (validCountry(COUNTRY)) {
let rs = fs.createReadStream(`${process.argv[2]}`);
rs.pipe(setupPipeline(COUNTRY));
}
cd data/countries
time parallel -j+0 curl -sS -o '{}.json' '"https://services.arcgis.com/P3ePLMYs2RVChkJx/ArcGIS/rest/services/World__Countries_Generalized_analysis_trim/FeatureServer/0/query?where=ISO_3DIGIT+%3D+%27{}%27&objectIds=&time=&geometry=&geometryType=esriGeometryEnvelope&inSR=&spatialRel=esriSpatialRelIntersects&resultType=none&distance=0.0&units=esriSRUnit_Meter&returnGeodetic=false&outFields=ISO_3DIGIT&returnGeometry=true&returnCentroid=false&featureEncoding=esriDefault&multipatchOption=xyFootprint&maxAllowableOffset=&geometryPrecision=&outSR=4326&datumTransformation=&applyVCSProjection=false&returnIdsOnly=false&returnUniqueIdsOnly=false&returnCountOnly=false&returnExtentOnly=false&returnQueryGeometry=false&returnDistinctValues=false&cacheHint=false&orderByFields=&groupByFieldsForStatistics=&outStatistics=&having=&resultOffset=&resultRecordCount=&returnZ=false&returnM=false&returnExceededLimitFeatures=true&quantizationParameters=&sqlFormat=none&f=geojson&token="' :::: <(cat list.txt)
cd data
curl -sS "https://services.arcgis.com/P3ePLMYs2RVChkJx/ArcGIS/rest/services/World__Countries_Generalized_analysis_trim/FeatureServer/0/query?where=1%3D1&objectIds=&time=&geometry=&geometryType=esriGeometryEnvelope&inSR=&spatialRel=esriSpatialRelIntersects&resultType=none&distance=0.0&units=esriSRUnit_Meter&returnGeodetic=false&outFields=ISO_3DIGIT&returnGeometry=false&returnCentroid=false&featureEncoding=esriDefault&multipatchOption=xyFootprint&maxAllowableOffset=&geometryPrecision=&outSR=&datumTransformation=&applyVCSProjection=false&returnIdsOnly=false&returnUniqueIdsOnly=false&returnCountOnly=false&returnExtentOnly=false&returnQueryGeometry=false&returnDistinctValues=true&cacheHint=false&orderByFields=&groupByFieldsForStatistics=&outStatistics=&having=&resultOffset=&resultRecordCount=&returnZ=false&returnM=false&returnExceededLimitFeatures=true&quantizationParameters=&sqlFormat=none&f=pjson&token=" | jq ' map(.attributes.ISO_3DIGIT)' > countries_iso_codes.json
// https://en.wikipedia.org/wiki/ISO_3166-1_alpha-3
// You should install jq for getting a plain text list of countries
mkdir -p data/countries
cd data/countries
curl -sS "https://services.arcgis.com/P3ePLMYs2RVChkJx/ArcGIS/rest/services/World__Countries_Generalized_analysis_trim/FeatureServer/0/query?where=1%3D1&objectIds=&time=&geometry=&geometryType=esriGeometryEnvelope&inSR=&spatialRel=esriSpatialRelIntersects&resultType=none&distance=0.0&units=esriSRUnit_Meter&returnGeodetic=false&outFields=ISO_3DIGIT&returnGeometry=false&returnCentroid=false&featureEncoding=esriDefault&multipatchOption=xyFootprint&maxAllowableOffset=&geometryPrecision=&outSR=&datumTransformation=&applyVCSProjection=false&returnIdsOnly=false&returnUniqueIdsOnly=false&returnCountOnly=false&returnExtentOnly=false&returnQueryGeometry=false&returnDistinctValues=true&cacheHint=false&orderByFields=&groupByFieldsForStatistics=&outStatistics=&having=&resultOffset=&resultRecordCount=&returnZ=false&returnM=false&returnExceededLimitFeatures=true&quantizationParameters=&sqlFormat=none&f=pjson&token=" | jq -r '.features[] | .attributes.ISO_3DIGIT' > list.txt
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment