url_checker.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "c22e6b6c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://data.pmel.noaa.gov/aclim/las/UI.html\n",
      "http://portal.aoos.org/\n",
      "https://portal.atn.ioos.us/\n",
      "https://portal.aoos.org\n",
      "https://data.pmel.noaa.gov/alamo/erddap/index.html\n",
      "https://arcticdata.io \n",
      "https://psl.noaa.gov/forecasts/seaice/\n",
      "https://argo.ucsd.edu/data/data-visualizations/\n",
      "https://psl.noaa.gov/atomic/\n",
      "https://psl.noaa.gov/map/time_plot/\n",
      "https://portal.aodn.org.au\n",
      "https://www.bco-dmo.org\n",
      " https://www.bodc.ac.uk \n",
      "https://www.integratedecosystemassessment.noaa.gov/regions/california-current/california-current-marine-heatwave-tracker-blobtracker\n"
     ]
    },
    {
     "ename": "HTTPError",
     "evalue": "HTTP Error 502: Bad Gateway",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mHTTPError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[21], line 17\u001b[0m\n\u001b[0;32m     14\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m resource \u001b[38;5;129;01min\u001b[39;00m data_json[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mlists\u001b[39m\u001b[38;5;124m'\u001b[39m]:\n\u001b[0;32m     16\u001b[0m     \u001b[38;5;28mprint\u001b[39m(resource[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124murl\u001b[39m\u001b[38;5;124m'\u001b[39m])\n\u001b[1;32m---> 17\u001b[0m     code \u001b[38;5;241m=\u001b[39m \u001b[43murllib\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\u001b[43mresource\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43murl\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241m.\u001b[39mgetcode()\n\u001b[0;32m     19\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m code \u001b[38;5;241m!=\u001b[39m \u001b[38;5;241m200\u001b[39m:\n\u001b[0;32m     20\u001b[0m         string \u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00murl\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m does not resolve. Code \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mcode\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n",
      "File \u001b[1;32m~\\programs\\Miniforge\\envs\\sandbox\\lib\\urllib\\request.py:216\u001b[0m, in \u001b[0;36murlopen\u001b[1;34m(url, data, timeout, cafile, capath, cadefault, context)\u001b[0m\n\u001b[0;32m    214\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m    215\u001b[0m     opener \u001b[38;5;241m=\u001b[39m _opener\n\u001b[1;32m--> 216\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mopener\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen\u001b[49m\u001b[43m(\u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32m~\\programs\\Miniforge\\envs\\sandbox\\lib\\urllib\\request.py:525\u001b[0m, in \u001b[0;36mOpenerDirector.open\u001b[1;34m(self, fullurl, data, timeout)\u001b[0m\n\u001b[0;32m    523\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m processor \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprocess_response\u001b[38;5;241m.\u001b[39mget(protocol, []):\n\u001b[0;32m    524\u001b[0m     meth \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mgetattr\u001b[39m(processor, meth_name)\n\u001b[1;32m--> 525\u001b[0m     response \u001b[38;5;241m=\u001b[39m \u001b[43mmeth\u001b[49m\u001b[43m(\u001b[49m\u001b[43mreq\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mresponse\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    527\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m response\n",
      "File \u001b[1;32m~\\programs\\Miniforge\\envs\\sandbox\\lib\\urllib\\request.py:634\u001b[0m, in \u001b[0;36mHTTPErrorProcessor.http_response\u001b[1;34m(self, request, response)\u001b[0m\n\u001b[0;32m    631\u001b[0m \u001b[38;5;66;03m# According to RFC 2616, \"2xx\" code indicates that the client's\u001b[39;00m\n\u001b[0;32m    632\u001b[0m \u001b[38;5;66;03m# request was successfully received, understood, and accepted.\u001b[39;00m\n\u001b[0;32m    633\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;241m200\u001b[39m \u001b[38;5;241m<\u001b[39m\u001b[38;5;241m=\u001b[39m code \u001b[38;5;241m<\u001b[39m \u001b[38;5;241m300\u001b[39m):\n\u001b[1;32m--> 634\u001b[0m     response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mparent\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43merror\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m    635\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mhttp\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mresponse\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcode\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmsg\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mhdrs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    637\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m response\n",
      "File \u001b[1;32m~\\programs\\Miniforge\\envs\\sandbox\\lib\\urllib\\request.py:563\u001b[0m, in \u001b[0;36mOpenerDirector.error\u001b[1;34m(self, proto, *args)\u001b[0m\n\u001b[0;32m    561\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m http_err:\n\u001b[0;32m    562\u001b[0m     args \u001b[38;5;241m=\u001b[39m (\u001b[38;5;28mdict\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mdefault\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mhttp_error_default\u001b[39m\u001b[38;5;124m'\u001b[39m) \u001b[38;5;241m+\u001b[39m orig_args\n\u001b[1;32m--> 563\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_chain\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32m~\\programs\\Miniforge\\envs\\sandbox\\lib\\urllib\\request.py:496\u001b[0m, in \u001b[0;36mOpenerDirector._call_chain\u001b[1;34m(self, chain, kind, meth_name, *args)\u001b[0m\n\u001b[0;32m    494\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m handler \u001b[38;5;129;01min\u001b[39;00m handlers:\n\u001b[0;32m    495\u001b[0m     func \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mgetattr\u001b[39m(handler, meth_name)\n\u001b[1;32m--> 496\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    497\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m result \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m    498\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m result\n",
      "File \u001b[1;32m~\\programs\\Miniforge\\envs\\sandbox\\lib\\urllib\\request.py:643\u001b[0m, in \u001b[0;36mHTTPDefaultErrorHandler.http_error_default\u001b[1;34m(self, req, fp, code, msg, hdrs)\u001b[0m\n\u001b[0;32m    642\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mhttp_error_default\u001b[39m(\u001b[38;5;28mself\u001b[39m, req, fp, code, msg, hdrs):\n\u001b[1;32m--> 643\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m HTTPError(req\u001b[38;5;241m.\u001b[39mfull_url, code, msg, hdrs, fp)\n",
      "\u001b[1;31mHTTPError\u001b[0m: HTTP Error 502: Bad Gateway"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import json\n",
    "import urllib\n",
    "\n",
    "data_url = \"https://github.com/NOAA-PSL/CEFI-info-hub-list/raw/main/data/cefi_list.json\"\n",
    "\n",
    "# store the response of URL\n",
    "response = urllib.request.urlopen(data_url)\n",
    "  \n",
    "# storing the JSON response \n",
    "# from url in data\n",
    "data_json = json.loads(response.read())\n",
    "\n",
    "for resource in data_json['lists']:\n",
    "    \n",
    "    print(resource['url'])\n",
    "    code = urllib.request.urlopen(resource['url']).getcode()\n",
    "\n",
    "    if code != 200:\n",
    "        string = f\"{url} does not resolve. Code {code}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9e20afe8",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}