Problematic ArcGIS Open Data Portals

pick-problematic.js

const fs = require('fs');
const rp = require('request-promise');
const Promise = require('bluebird');
const _ = require('lodash');
const cheerio = require('cheerio');

let portals = JSON.parse(fs.readFileSync('open-data-sources.json'));
let arcgisPortals = _.chain(portals)
                      .filter(portal => {
                        return portal.fields.url.indexOf('arcgis') > -1 && portal.fields.has_issue !== 'TRUE';
                      })
                      .map(portal => {
                        return { name: portal.fields.name, url: portal.fields.url };
                      })
                      .value();

let requests = _.map(arcgisPortals, portal => {
  return rp({
    method: 'GET',
    uri: `${portal.url}/datasets`,
    json: true
  })
  .then(result => {
    if (!result.metadata) {
      let $ = cheerio.load(result);

      // if the url is redirected to http://opendata.arcgis.com/about,
      // the returned html should include the title 'Esri | ArcGIS Open Data'.
      // However, this may not be 100% correct.
      if ($('title').text() === 'Esri | ArcGIS Open Data') {
        return portal;
      }

      return null;
    }

    return null;
  })
  .catch(err => {
    // this is an unreachable portal, probably 404
    return portal;
  });
});

Promise.all(requests)
  .then(results => {
    _.remove(results, _.isNull);
    fs.writeFileSync('problematic-arcgis-portals.json', JSON.stringify(results));
  });

portal-list.json

[
  {
    "name": "משרדהחקלאותופיתוחהכפר-מידעגיאוגרפיחופשי",
    "url": "http://data.moag.opendata.arcgis.com"
  },
  {
    "name": "Bovendams Open Data",
    "url": "http://waterschap.bovendams.opendata.arcgis.com"
  },
  {
    "name": "Example Site",
    "url": "http://data19.dcdev.opendata.arcgis.com"
  },
  {
    "name": "Liz_OD",
    "url": "http://lizod.ess.opendata.arcgis.com"
  },
  {
    "name": "Placer County Open Data",
    "url": "http://placergis.placercounty.opendata.arcgis.com"
  },
  {
    "name": "HNIT-BALTICUAB Arc GIS Onlinedemopaskyra",
    "url": "http://mindaugasr.hb.opendata.arcgis.com"
  },
  {
    "name": "City of Redmond Parks",
    "url": "http://redmondparks.redmondoregon.opendata.arcgis.com"
  },
  {
    "name": "Officeofthe CAGeographic Information Officer",
    "url": "http://data.california.opendata.arcgis.com"
  },
  {
    "name": "Orange, Ca - Gis Data",
    "url": "http://cityoforange.maps.arcgis.com/home/index.html"
  },
  {
    "name": "Skogsdataportalen",
    "url": "http://skogsdataportalen.esri-skog.opendata.arcgis.com"
  },
  {
    "name": "TUC Demo Open Data",
    "url": "http://tuc2014.tuc.opendata.arcgis.com"
  },
  {
    "name": "Monterey County, Ca - Gis Open Data",
    "url": "http://montereycountyopendata.montereyco.opendata.arcgis.com"
  },
  {
    "name": "Federating",
    "url": "http://federating.dcdev.opendata.arcgis.com"
  },
  {
    "name": "Villede Saint-Quentin",
    "url": "http://data.saint-quentin.opendata.arcgis.com"
  },
  {
    "name": "Utah's State Geographic Information Database(AGRC)",
    "url": "http://test.utah.opendata.arcgis.com"
  },
  {
    "name": "Conseil Généraldu Haut-Rhin",
    "url": "http://data.cg68.opendata.arcgis.com"
  },
  {
    "name": "Welcometothe Geodev Organization",
    "url": "http://data.edn.opendata.arcgis.com"
  },
  {
    "name": "City of Bradford MDC",
    "url": "http://data.cbmdc.opendata.arcgis.com"
  },
  {
    "name": "Arc GIS Taiwan Open Data",
    "url": "http://government.ernest.opendata.arcgis.com"
  },
  {
    "name": "Hoogheemraadschap Hollands Noorderkwartier",
    "url": "http://data.hhnk.opendata.arcgis.com"
  },
  {
    "name": "UMN Parkingand Transportation Services-Open Traffic Data Portal",
    "url": "http://opendata.arcgis.co"
  },
  {
    "name": "Kurikankaupunki",
    "url": "http://data.kurka.opendata.arcgis.com"
  },
  {
    "name": "Beta Departmentof Housingand Urban Development",
    "url": "http://betaegis.hud.opendata.arcgis.com"
  },
  {
    "name": "ESRI Support Services",
    "url": "http://data4.ess.opendata.arcgis.com"
  },
  {
    "name": "Monsite Open Data",
    "url": "http://monsite.demoesrifrance2.opendata.arcgis.com"
  },
  {
    "name": "DGPPF",
    "url": "http://dgppf.conadesuca.opendata.arcgis.com"
  },
  {
    "name": "Mary's ESS Open Data Site",
    "url": "http://data91.ess.opendata.arcgis.com"
  },
  {
    "name": "Portal Geografico",
    "url": "http://data.defensa-esri-co.opendata.arcgis.com"
  },
  {
    "name": "FEROPENDATA",
    "url": "http://fer.sop.opendata.arcgis.com"
  },
  {
    "name": "City of Sugar Land Open Data",
    "url": "http://public2.cosl.opendata.arcgis.com"
  },
  {
    "name": "Open Data",
    "url": "http://enrics.boprc.opendata.arcgis.com"
  },
  {
    "name": "Esri Thailand(Demo Site)Open Data",
    "url": "http://opendata.esrith-demo.opendata.arcgis.com"
  },
  {
    "name": "Esri Australia Local Government",
    "url": "http://data.esriau-localgovt.opendata.arcgis.com"
  },
  {
    "name": "JULIA5",
    "url": "http://data27.ess.opendata.arcgis.com"
  },
  {
    "name": "griffen Syte",
    "url": "http://data25.ess.opendata.arcgis.com"
  },
  {
    "name": "Montgomery County, Al - Arcgis",
    "url": "http://gis.montgomeryal.gov/arcgis/rest/services"
  },
  {
    "name": "Mountain Bike Routes",
    "url": "http://mtb.esrifinland.opendata.arcgis.com"
  },
  {
    "name": "SIGU-TOPOGRAPHIE",
    "url": "http://data.sigu.opendata.arcgis.com"
  },
  {
    "name": "E Ng Ibrahim Jarrar",
    "url": "http://jarrar.jordantraining.opendata.arcgis.com"
  },
  {
    "name": "Departmentof Conservation Open Data-Beta",
    "url": "http://geo.deptconservation.opendata.arcgis.com"
  },
  {
    "name": "JOHNPLUNKETT",
    "url": "http://data1.spt-eaglegis.opendata.arcgis.com"
  },
  {
    "name": "CIE-CAÑA",
    "url": "http://map.conadesuca.opendata.arcgis.com"
  },
  {
    "name": "Gemeente Arnhem",
    "url": "http://gemeentearnhem2.esrinederland.opendata.arcgis.com"
  },
  {
    "name": "EU Dev Summit",
    "url": "http://opendatalondon.techresearch.opendata.arcgis.com"
  },
  {
    "name": "geohub. Barranquilla",
    "url": "http://geohubbarranquilla.gov-esri-co.opendata.arcgis.com"
  },
  {
    "name": "City of Greenwood Village",
    "url": "http://cityofgreenwoodvillage.gwv.opendata.arcgis.com"
  },
  {
    "name": "Gemeente Wageningen",
    "url": "http://data.wageningen.opendata.arcgis.com"
  },
  {
    "name": "FAO Swalim",
    "url": "http://faoswalim.esri-ea.opendata.arcgis.com"
  },
  {
    "name": "QuảnlíMôi Trường",
    "url": "http://ntmx.qlmt.opendata.arcgis.com"
  },
  {
    "name": "Datos Abiertos IDIGER",
    "url": "http://mapas.idiger.opendata.arcgis.com"
  },
  {
    "name": "Soporte Esri Colombia",
    "url": "http://esricolombia.petmin-esri-co.opendata.arcgis.com"
  },
  {
    "name": "Beautify Our Neighborhood",
    "url": "http://data.gspace.opendata.arcgis.com"
  },
  {
    "name": "DBSNE Open Data Site-Copy",
    "url": "http://open.dbsne.opendata.arcgis.com"
  },
  {
    "name": "Northumberland County Council",
    "url": "http://openmapdata.northumberland.opendata.arcgis.com"
  },
  {
    "name": "互動國際數位台中 Open Data",
    "url": "http://test.ernest.opendata.arcgis.com"
  },
  {
    "name": "Open Data Stadt Chemnitz",
    "url": "http://portal.chemnitz.opendata.arcgis.com/"
  },
  {
    "name": "City of Sugar Land Open GIS Data",
    "url": "http://public1.cosl.opendata.arcgis.com"
  },
  {
    "name": "City of Tucson",
    "url": "http://data1.cotgis.opendata.arcgis.com"
  },
  {
    "name": "LCG Open Data Portal2016",
    "url": "http://downloads.lcg.opendata.arcgis.com"
  },
  {
    "name": "Management& Developmentfor Health",
    "url": "http://mdh.esri-ea.opendata.arcgis.com"
  },
  {
    "name": "City of Melbourne Open Data Site",
    "url": "http://com.esriau.opendata.arcgis.com"
  },
  {
    "name": "Open Data Day Demo",
    "url": "http://opendataday.geotec.opendata.arcgis.com"
  },
  {
    "name": "AAbrooks WAT",
    "url": "http://aabrooks2.dcdev.opendata.arcgis.com"
  },
  {
    "name": "Agic Geodata Portal (azgeo)",
    "url": "https://arcgis2.geo.az.gov/portal/"
  },
  {
    "name": "USDANASS",
    "url": "http://nass.dcdev.opendata.arcgis.com"
  },
  {
    "name": "Joan's Random Maps Open Data",
    "url": "http://randommaps.joan.opendata.arcgis.com"
  },
  {
    "name": "LCG2015OPENDATASITE",
    "url": "http://gis.lcg.opendata.arcgis.com"
  },
  {
    "name": "Shondra's Open Data",
    "url": "http://data32.ess.opendata.arcgis.com"
  },
  {
    "name": "Datos Abiertos Cliente",
    "url": "http://datosprueba.esri-colombia.opendata.arcgis.com"
  },
  {
    "name": "POLIOADMINBOUNDARIES",
    "url": "http://data.ebolaresponse.opendata.arcgis.com"
  },
  {
    "name": "Alaska Divisonof Oiland Gas Open Data",
    "url": "http://dog.soa-dnr.opendata.arcgis.com"
  },
  {
    "name": "Alaska Departmentof Natural Resources",
    "url": "http://adnr.soa-dnr.opendata.arcgis.com"
  },
  {
    "name": "CCGIS-Cumberland County GIS",
    "url": "http://ccgis.cumberlandgis.opendata.arcgis.com"
  },
  {
    "name": "Jill E's Open Data",
    "url": "http://data24.ess.opendata.arcgis.com"
  },
  {
    "name": "Cambridge,Ontario",
    "url": "http://data.cityofcambridge.opendata.arcgis.com"
  },
  {
    "name": "Regional Districtof East Kootenay Open GIS Data Site",
    "url": "http://regionaldistrictofeastkootenayopengisdatasite.rdek.opendata.arcgis.com"
  },
  {
    "name": "Style Testing",
    "url": "http://styletesting-1.dcgis.opendata.arcgis.com"
  },
  {
    "name": "Test",
    "url": "http://test.rmw.opendata.arcgis.com"
  },
  {
    "name": "ESRICOLOMBIA",
    "url": "http://esrico.gov-esri-co.opendata.arcgis.com"
  },
  {
    "name": "Marg",
    "url": "http://marg.ess.opendata.arcgis.com"
  },
  {
    "name": "Stateof Idaho Open Data",
    "url": "http://portal.idaho.opendata.arcgis.com"
  },
  {
    "name": "Kern County, Ca - Arcgis",
    "url": "http://maps.co.kern.ca.us/arcgis/rest/services"
  },
  {
    "name": "Bexar County Open Data",
    "url": "http://gis-old.bexar.opendata.arcgis.com"
  },
  {
    "name": "MORPC Open Data",
    "url": "http://data.morpc.opendata.arcgis.com"
  },
  {
    "name": "Open Data Catalog Dev",
    "url": "http://dev.fedmaps.opendata.arcgis.com"
  },
  {
    "name": "San Bernardino County Departmentof Public Works Open Data",
    "url": "http://gis.sbcountydpw.opendata.arcgis.com"
  },
  {
    "name": "Comercioy Negocios-Esri Colombia",
    "url": "http://comercio.negocios-esri-co.opendata.arcgis.com"
  },
  {
    "name": "Portal Geografico",
    "url": "http://pruabaopendata.defensa-esri-co.opendata.arcgis.com"
  },
  {
    "name": "Evansville Vanderburgh County Area Plan",
    "url": "http://data.evansvilleapc.opendata.arcgis.com"
  },
  {
    "name": "Bexar County",
    "url": "http://data.bexar.opendata.arcgis.com"
  },
  {
    "name": "Infraestructurade Datos Espaciales",
    "url": "http://data.igac-ide.opendata.arcgis.com"
  },
  {
    "name": "ESRI Chile",
    "url": "http://2014.esrichile.opendata.arcgis.com"
  },
  {
    "name": "St. Louis County GIS Service Center Open Data-Copy",
    "url": "http://openstlco.stlcogis.opendata.arcgis.com"
  },
  {
    "name": "City of Launceston-Open Data",
    "url": "http://lcc.launceston.opendata.arcgis.com"
  },
  {
    "name": "Municipiode Medellín",
    "url": "http://data.m-medellin.opendata.arcgis.com"
  },
  {
    "name": "Costello Inc.",
    "url": "http://data.costello.opendata.arcgis.com"
  },
  {
    "name": "Hastings District Council Open Data",
    "url": "http://data.hdcgis.opendata.arcgis.com"
  }
]