crawl-cdn-hit-rate-most-common-pages-GET.py

import concurrent.futures
import random
import time
import statistics
from collections import defaultdict
from urllib.parse import urlparse

import requests
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry


def requests_retry_session(
    retries=3, backoff_factor=0.3, status_forcelist=(500, 502, 504)
):
    """Opinionated wrapper that creates a requests session with a
    HTTPAdapter that sets up a Retry policy that includes connection
    retries.

    If you do the more naive retry by simply setting a number. E.g.::

        adapter = HTTPAdapter(max_retries=3)

    then it will raise immediately on any connection errors.
    Retrying on connection errors guards better on unpredictable networks.
    From http://docs.python-requests.org/en/master/api/?highlight=retries#requests.adapters.HTTPAdapter
    it says: "By default, Requests does not retry failed connections."

    The backoff_factor is documented here:
    https://urllib3.readthedocs.io/en/latest/reference/urllib3.util.html#urllib3.util.retry.Retry
    A default of retries=3 and backoff_factor=0.3 means it will sleep like::

        [0.3, 0.6, 1.2]
    """  # noqa
    session = requests.Session()
    retry = Retry(
        total=retries,
        read=retries,
        connect=retries,
        backoff_factor=backoff_factor,
        status_forcelist=status_forcelist,
    )
    adapter = HTTPAdapter(max_retries=retry)
    session.mount("http://", adapter)
    session.mount("https://", adapter)
    return session


def probe(url):
    t0 = time.time()
    response = requests.get(url)
    t1 = time.time()
    response.raise_for_status()
    value = response.headers["x-cache"]
    print("{:100} {} Took: {:.3f}s".format(urlparse(url).path, value, t1 - t0))
    return value, t1 - t0


def start(samplesize):
    # root = "https://developer.mozilla.org/en-US/docs/all"
    # urls = get_urls(root, 1500)
    # print("Gathered", len(urls), "URLs")
    import json
    urls = json.load(open('most-common-urls.json'))

    results = {}
    with concurrent.futures.ThreadPoolExecutor(max_workers=3) as executor:
        futures = {}
        for url in random.sample(urls, min(len(urls), samplesize)):
            if '://' not in url:
                url = "https://developer.mozilla.org" + url
            futures[executor.submit(probe, url)] = url
        for future in concurrent.futures.as_completed(futures):
            result = future.result()
            results[futures[future]] = result

    misses = hits = 0
    times = defaultdict(list)
    for value, took in results.values():
        times[value].append(took)
        if "Miss from cloudfront" in value:
            misses += 1
        elif "Hit from cloudfront" in value:
            hits += 1
        else:
            raise NotImplementedError(value)

    print("HIT RATIO: {:.1f}%".format(100 * hits / (hits + misses)))
    print()
    for value, tooks in times.items():
        print("MEDIAN TIME FOR {!r}: {:.2f}s (# {})".format(
            value, statistics.median(tooks), len(tooks)
        ))




# def sanitytest():
#     import time

#     for i in range(5):
#         r = session.head(
#             "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Math/hypot"
#         )
#         print(r.headers["x-cache"])
#         time.sleep(1)


# sanitytest()
start(500)

most-common-urls.json

[
  "https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest",
  "https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API/Using_Fetch",
  "https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API",
  "https://developer.mozilla.org/en-US/docs/Web/API/EventTarget/addEventListener",
  "https://developer.mozilla.org/en-US/docs/Web/API/History_API",
  "https://developer.mozilla.org/en-US/docs/Web/API/Document/getElementById",
  "https://developer.mozilla.org/en-US/docs/Web/API/Document/querySelector",
  "https://developer.mozilla.org/en-US/docs/Web/API/Window/localStorage",
  "https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest/Using_XMLHttpRequest",
  "https://developer.mozilla.org/en-US/docs/Web/API/WindowBase64/Base64_encoding_and_decoding",
  "https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest",
  "https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API/Using_Fetch",
  "https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API",
  "https://developer.mozilla.org/en-US/docs/Web/API/EventTarget/addEventListener",
  "https://developer.mozilla.org/en-US/docs/Web/API/History_API",
  "https://developer.mozilla.org/en-US/docs/Web/API/Document/getElementById",
  "https://developer.mozilla.org/en-US/docs/Web/API/Document/querySelector",
  "https://developer.mozilla.org/en-US/docs/Web/API/Window/localStorage",
  "https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest/Using_XMLHttpRequest",
  "https://developer.mozilla.org/en-US/docs/Web/API/WindowBase64/Base64_encoding_and_decoding",
  "https://developer.mozilla.org/en-US/docs/Web/API/Window/postMessage",
  "https://developer.mozilla.org/en-US/docs/Web/API/Element/classList",
  "https://developer.mozilla.org/en-US/docs/Web/API/FileReader",
  "https://developer.mozilla.org/en-US/docs/Web/API/Window/open",
  "https://developer.mozilla.org/en-US/docs/Web/API/Blob",
  "https://developer.mozilla.org/en-US/docs/Web/API/WebSocket",
  "https://developer.mozilla.org/en-US/docs/Web/API/WebSockets_API/Writing_WebSocket_client_applications",
  "https://developer.mozilla.org/en-US/docs/Web/API/WebSockets_API",
  "https://developer.mozilla.org/en-US/docs/Web/API/File",
  "https://developer.mozilla.org/en-US/docs/Web/API/FormData",
  "https://developer.mozilla.org/en-US/docs/Web/API/Document/createElement",
  "https://developer.mozilla.org/en-US/docs/Web/API/FormData/Using_FormData_Objects",
  "https://developer.mozilla.org/en-US/docs/Web/API/Window/location",
  "https://developer.mozilla.org/en-US/docs/Web/API/WebGL_API",
  "https://developer.mozilla.org/en-US/docs/Web/API/Window/sessionStorage",
  "https://developer.mozilla.org/en-US/docs/Web/API/Element/scrollIntoView",
  "https://developer.mozilla.org/en-US/docs/Web/API/Document_Object_Model",
  "https://developer.mozilla.org/en-US/docs/Web/API/Document/getElementsByClassName",
  "https://developer.mozilla.org/en-US/docs/Web/API/Element/getBoundingClientRect",
  "https://developer.mozilla.org/en-US/docs/Web/API/Console/log",
  "https://developer.mozilla.org/en-US/docs/Web/API/Canvas_API/Tutorial/Drawing_shapes",
  "https://developer.mozilla.org/en-US/docs/Web/API/Geolocation_API",
  "https://developer.mozilla.org/en-US/docs/Web/API/Element",
  "https://developer.mozilla.org/en-US/docs/Web/API/HTML_Drag_and_Drop_API",
  "https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Using_web_workers",
  "https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope/setTimeout",
  "https://developer.mozilla.org/en-US/docs/Web/API/Document/cookie",
  "https://developer.mozilla.org/en-US/docs/Web/API/Event",
  "https://developer.mozilla.org/en-US/docs/Web/API/window/requestAnimationFrame",
  "https://developer.mozilla.org/en-US/docs/Web/API/Canvas_API/Tutorial",
  "https://developer.mozilla.org/en-US/docs/Web/API/Canvas_API",
  "https://developer.mozilla.org/en-US/docs/Web/API/MutationObserver",
  "https://developer.mozilla.org/en-US/docs/Web/API/Document/querySelectorAll",
  "https://developer.mozilla.org/en-US/docs/Web/API/Service_Worker_API",
  "https://developer.mozilla.org/en-US/docs/Web/API/Element/setAttribute",
  "https://developer.mozilla.org/en-US/docs/Web/API/Document_Object_Model/Introduction",
  "https://developer.mozilla.org/en-US/docs/Web/API/FormData/append",
  "https://developer.mozilla.org/en-US/docs/Web/API/File/Using_files_from_web_applications",
  "https://developer.mozilla.org/en-US/docs/Web/API/IndexedDB_API",
  "https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest/send",
  "https://developer.mozilla.org/en-US/docs/Web/API/Element/innerHTML",
  "https://developer.mozilla.org/en-US/docs/Web/API/Touch_events",
  "https://developer.mozilla.org/en-US/docs/Web/API/FileReader/readAsDataURL",
  "https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices/getUserMedia",
  "https://developer.mozilla.org/en-US/docs/Web/API/Service_Worker_API/Using_Service_Workers",
  "https://developer.mozilla.org/en-US/docs/Web/API/Web_Audio_API",
  "https://developer.mozilla.org/en-US/docs/Web/API/EventTarget/removeEventListener",
  "https://developer.mozilla.org/en-US/docs/Web/API/Node/appendChild",
  "https://developer.mozilla.org/en-US/docs/Web/API/GlobalEventHandlers/onload",
  "https://developer.mozilla.org/en-US/docs/Web/API/HTMLElement/style",
  "https://developer.mozilla.org/en-US/docs/Web/API/Document",
  "https://developer.mozilla.org/en-US/docs/Web/API/Request",
  "https://developer.mozilla.org/en-US/docs/Web/API/Fullscreen_API",
  "https://developer.mozilla.org/en-US/docs/Web/API/notification",
  "https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope/setInterval",
  "https://developer.mozilla.org/en-US/docs/Web/API/Web_Storage_API",
  "https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest/open",
  "https://developer.mozilla.org/en-US/docs/Web/API/Event/preventDefault",
  "https://developer.mozilla.org/en-US/docs/Web/API/URL/createObjectURL",
  "https://developer.mozilla.org/en-US/docs/Web/API/Response",
  "https://developer.mozilla.org/en-US/docs/Web/API/HTMLCanvasElement/toDataURL",
  "https://developer.mozilla.org/en-US/docs/Web/API/Window",
  "https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest/Synchronous_and_Asynchronous_Requests",
  "https://developer.mozilla.org/en-US/docs/Web/API/KeyboardEvent",
  "https://developer.mozilla.org/en-US/docs/Web/API/Element/getAttribute",
  "https://developer.mozilla.org/en-US/docs/Web/API/Document/execCommand",
  "https://developer.mozilla.org/en-US/docs/Web/API/Canvas_API/Tutorial/Basic_usage",
  "https://developer.mozilla.org/en-US/docs/Web/API/MouseEvent",
  "https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest/Sending_and_Receiving_Binary_Data",
  "https://developer.mozilla.org/en-US/docs/Web/API/CanvasRenderingContext2D/drawImage",
  "https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest",
  "https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API/Using_Fetch",
  "https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API",
  "https://developer.mozilla.org/en-US/docs/Web/API/EventTarget/addEventListener",
  "https://developer.mozilla.org/en-US/docs/Web/API/History_API",
  "https://developer.mozilla.org/en-US/docs/Web/API/Document/getElementById",
  "https://developer.mozilla.org/en-US/docs/Web/API/Document/querySelector",
  "https://developer.mozilla.org/en-US/docs/Web/API/Window/localStorage",
  "https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest/Using_XMLHttpRequest",
  "https://developer.mozilla.org/en-US/docs/Web/API/WindowBase64/Base64_encoding_and_decoding",
  "https://developer.mozilla.org/en-US/docs/Web/API/Window/postMessage",
  "https://developer.mozilla.org/en-US/docs/Web/API/Element/classList",
  "https://developer.mozilla.org/en-US/docs/Web/API/FileReader",
  "https://developer.mozilla.org/en-US/docs/Web/API/Window/open",
  "https://developer.mozilla.org/en-US/docs/Web/API/Blob",
  "https://developer.mozilla.org/en-US/docs/Web/API/WebSocket",
  "https://developer.mozilla.org/en-US/docs/Web/API/WebSockets_API/Writing_WebSocket_client_applications",
  "https://developer.mozilla.org/en-US/docs/Web/API/WebSockets_API",
  "https://developer.mozilla.org/en-US/docs/Web/API/File",
  "https://developer.mozilla.org/en-US/docs/Web/API/FormData",
  "https://developer.mozilla.org/en-US/docs/Web/API/Document/createElement",
  "https://developer.mozilla.org/en-US/docs/Web/API/FormData/Using_FormData_Objects",
  "https://developer.mozilla.org/en-US/docs/Web/API/Window/location",
  "https://developer.mozilla.org/en-US/docs/Web/API/WebGL_API",
  "https://developer.mozilla.org/en-US/docs/Web/API/Window/sessionStorage",
  "https://developer.mozilla.org/en-US/docs/Web/API/Element/scrollIntoView",
  "https://developer.mozilla.org/en-US/docs/Web/API/Document_Object_Model",
  "https://developer.mozilla.org/en-US/docs/Web/API/Document/getElementsByClassName",
  "https://developer.mozilla.org/en-US/docs/Web/API/Element/getBoundingClientRect",
  "https://developer.mozilla.org/en-US/docs/Web/API/Console/log",
  "https://developer.mozilla.org/en-US/docs/Web/API/Canvas_API/Tutorial/Drawing_shapes",
  "https://developer.mozilla.org/en-US/docs/Web/API/Geolocation_API",
  "https://developer.mozilla.org/en-US/docs/Web/API/Element",
  "https://developer.mozilla.org/en-US/docs/Web/API/HTML_Drag_and_Drop_API",
  "https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Using_web_workers",
  "https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope/setTimeout",
  "https://developer.mozilla.org/en-US/docs/Web/API/Document/cookie",
  "https://developer.mozilla.org/en-US/docs/Web/API/Event",
  "https://developer.mozilla.org/en-US/docs/Web/API/window/requestAnimationFrame",
  "https://developer.mozilla.org/en-US/docs/Web/API/Canvas_API/Tutorial",
  "https://developer.mozilla.org/en-US/docs/Web/API/Canvas_API",
  "https://developer.mozilla.org/en-US/docs/Web/API/MutationObserver",
  "https://developer.mozilla.org/en-US/docs/Web/API/Document/querySelectorAll",
  "https://developer.mozilla.org/en-US/docs/Web/API/Service_Worker_API",
  "https://developer.mozilla.org/en-US/docs/Web/API/Element/setAttribute",
  "https://developer.mozilla.org/en-US/docs/Web/API/Document_Object_Model/Introduction",
  "https://developer.mozilla.org/en-US/docs/Web/API/FormData/append",
  "https://developer.mozilla.org/en-US/docs/Web/API/File/Using_files_from_web_applications",
  "https://developer.mozilla.org/en-US/docs/Web/API/IndexedDB_API",
  "https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest/send",
  "https://developer.mozilla.org/en-US/docs/Web/API/Element/innerHTML",
  "https://developer.mozilla.org/en-US/docs/Web/API/Touch_events",
  "https://developer.mozilla.org/en-US/docs/Web/API/FileReader/readAsDataURL",
  "https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices/getUserMedia",
  "https://developer.mozilla.org/en-US/docs/Web/API/Service_Worker_API/Using_Service_Workers",
  "https://developer.mozilla.org/en-US/docs/Web/API/Web_Audio_API",
  "https://developer.mozilla.org/en-US/docs/Web/API/EventTarget/removeEventListener",
  "https://developer.mozilla.org/en-US/docs/Web/API/Node/appendChild",
  "https://developer.mozilla.org/en-US/docs/Web/API/GlobalEventHandlers/onload",
  "https://developer.mozilla.org/en-US/docs/Web/API/HTMLElement/style",
  "https://developer.mozilla.org/en-US/docs/Web/API/Document",
  "https://developer.mozilla.org/en-US/docs/Web/API/Request",
  "https://developer.mozilla.org/en-US/docs/Web/API/Fullscreen_API",
  "https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope/setInterval",
  "https://developer.mozilla.org/en-US/docs/Web/API/Web_Storage_API",
  "https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest/open",
  "https://developer.mozilla.org/en-US/docs/Web/API/Event/preventDefault",
  "https://developer.mozilla.org/en-US/docs/Web/API/URL/createObjectURL",
  "https://developer.mozilla.org/en-US/docs/Web/API/Response",
  "https://developer.mozilla.org/en-US/docs/Web/API/HTMLCanvasElement/toDataURL",
  "https://developer.mozilla.org/en-US/docs/Web/API/Window",
  "https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest/Synchronous_and_Asynchronous_Requests",
  "https://developer.mozilla.org/en-US/docs/Web/API/KeyboardEvent",
  "https://developer.mozilla.org/en-US/docs/Web/API/Element/getAttribute",
  "https://developer.mozilla.org/en-US/docs/Web/API/Document/execCommand",
  "https://developer.mozilla.org/en-US/docs/Web/API/Canvas_API/Tutorial/Basic_usage",
  "https://developer.mozilla.org/en-US/docs/Web/API/MouseEvent",
  "https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest/Sending_and_Receiving_Binary_Data",
  "https://developer.mozilla.org/en-US/docs/Web/API/CanvasRenderingContext2D/drawImage",
  "https://developer.mozilla.org/en-US/docs/Web/API/Geolocation/getCurrentPosition",
  "https://developer.mozilla.org/en-US/docs/Web/API/Element/scrollTop",
  "https://developer.mozilla.org/en-US/docs/Web/API/WindowEventHandlers/onbeforeunload",
  "https://developer.mozilla.org/en-US/docs/Web/API/URLSearchParams",
  "https://developer.mozilla.org/en-US/docs/Web/API/Console",
  "https://developer.mozilla.org/en-US/docs/Web/API/Notifications_API/Using_the_Notifications_API",
  "https://developer.mozilla.org/en-US/docs/Web/API/Location/reload",
  "https://developer.mozilla.org/en-US/docs/Web/API/Web_Storage_API/Local_storage",
  "https://developer.mozilla.org/en-US/docs/Web/API/Geolocation_API",
  "https://developer.mozilla.org/en-US/docs/Web/API/Web_Storage_API/Local_storage",
  "https://developer.mozilla.org/en-US/docs/Web/API/Canvas_API/Tutorial/Using_images",
  "https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest/setRequestHeader",
  "https://developer.mozilla.org/en-US/docs/Web/API/WebSockets_API/Writing_WebSocket_servers",
  "https://developer.mozilla.org/en-US/docs/Web/API/Event/stopPropagation",
  "https://developer.mozilla.org/en-US/docs/Web/API/CanvasRenderingContext2D",
  "https://developer.mozilla.org/en-US/docs/Web/API/HTMLImageElement/Image",
  "https://developer.mozilla.org/en-US/docs/Web/API/Geolocation_API",
  "https://developer.mozilla.org/en-US/docs/Web/API/Web_Storage_API/Local_storage",
  "https://developer.mozilla.org/en-US/docs/Web/API/Web_Storage_API/Using_the_Web_Storage_API",
  "https://developer.mozilla.org/en-US/docs/Web/API/HTMLElement",
  "https://developer.mozilla.org/en-US/docs/Web/API/WebGL_API/Tutorial/Getting_started_with_WebGL",
  "https://developer.mozilla.org/en-US/docs/Web/API/KeyboardEvent/keyCode",
  "https://developer.mozilla.org/en-US/docs/Web/API/Web_Speech_API",
  "/en-US/",
  "https://developer.mozilla.org/en-US/docs/Web/CSS/Specificity",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/forEach",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/filter",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/map",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/find",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript",
  "https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS",
  "/zh-CN/",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/Reduce",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Date",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/splice",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/slice",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object/assign",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Map",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/includes",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Classes",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object/keys",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Functions/Arrow_functions",
  "https://developer.mozilla.org/en-US/docs/Web",
  "https://developer.mozilla.org/zh-CN/docs/Web/JavaScript",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/sort",
  "https://developer.mozilla.org/en-US/docs/Web/HTML/Element/input",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/typeof",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Template_literals",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/JSON/stringify",
  "https://developer.mozilla.org/en-US/docs/Learn",
  "https://developer.mozilla.org/en-US/docs/Web/HTML/Element",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/replace",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object/values",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/indexOf",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Errors/Not_defined",
  "https://developer.mozilla.org/en-US/docs/Web/HTTP/Status",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Errors/Unexpected_type",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Conditional_Operator",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/all",
  "https://developer.mozilla.org/zh-CN/docs/Web",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Set",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Spread_syntax",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Destructuring_assignment",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/split",
  "https://developer.mozilla.org/en-US/docs/Web/HTML",
  "https://developer.mozilla.org/en-US/docs/Web/HTML/Element/input/date",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/some",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/concat",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/async_function",
  "https://developer.mozilla.org/en-US/docs/Tools/Debugger/Source_map_errors",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/match",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object",
  "https://developer.mozilla.org/en-US/docs/Web/CSS",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/for...in",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/findIndex",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Math/random",
  "https://developer.mozilla.org/en-US/docs/Web/HTML/Element/input/file",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/import",
  "https://developer.mozilla.org/en-US/docs/Web/API",
  "https://developer.mozilla.org/en-US/docs/Learn/Getting_started_with_the_web/JavaScript_basics",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Working_with_Objects",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/A_re-introduction_to_JavaScript",
  "https://developer.mozilla.org/en-US/docs/Web/CSS/Reference",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/RegExp",
  "https://developer.mozilla.org/en-US/docs/Web/CSS/CSS_Flexible_Box_Layout/Aligning_Items_in_a_Flex_Container",
  "https://developer.mozilla.org/en-US/docs/Web/HTML/Element/input/checkbox",
  "https://developer.mozilla.org/en-US/docs/Web/HTML/Element/input/number",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/JSON/parse",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/export",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/push",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object/entries",
  "https://developer.mozilla.org/en-US/docs/Web/Progressive_web_apps",
  "https://developer.mozilla.org/en-US/docs/Web/Events",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/for...of",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Date/now",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Errors/Not_a_function",
  "https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Frame-Options",
  "https://developer.mozilla.org/en-US/docs/Learn/HTML/Forms/Form_validation",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/join",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Math",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/from",
  "https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Function/bind",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/every",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Date/parse",
  "https://developer.mozilla.org/es/docs/Learn/HTML/Multimedia_and_embedding/Images_in_HTML",
  "https://developer.mozilla.org/en-US/docs/Tools",
  "https://developer.mozilla.org/es/docs/HTML/HTML5/HTML5_lista_elementos",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/resolve",
  "https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/400",
  "https://developer.mozilla.org/en-US/docs/Web/HTML/Element/input/radio",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/parseInt",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/then",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/substring",
  "https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Type",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Number",
  "https://developer.mozilla.org/en-US/docs/Web/CSS/CSS_Flexible_Box_Layout/Basic_Concepts_of_Flexbox",
  "https://developer.mozilla.org/en-US/docs/Learn/Getting_started_with_the_web",
  "https://developer.mozilla.org/en-US/docs/Learn/HTML/Introduction_to_HTML",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Using_promises",
  "https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types",
  "https://developer.mozilla.org/en-US/docs/Web/CSS/linear-gradient",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object/hasOwnProperty",
  "https://developer.mozilla.org/zh-CN/docs/learn",
  "https://developer.mozilla.org/en-US/docs/Web/CSS/Media_Queries/Using_media_queries",
  "https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Allow-Origin",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Math/round",
  "https://developer.mozilla.org/en-US/docs/Web/CSS/box-shadow",
  "https://developer.mozilla.org/zh-CN/docs/Web/JavaScript/Reference/Errors/Not_defined",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference",
  "https://developer.mozilla.org/en-US/docs/Web/CSS/position",
  "https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/isArray",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Data_structures",
  "https://developer.mozilla.org/en-US/docs/Web/HTML/Element/select",
  "https://developer.mozilla.org/en-US/docs/Learn/HTML/Howto/Use_data_attributes",
  "https://developer.mozilla.org/en-US/docs/Learn/HTML/Forms/Sending_and_retrieving_form_data",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/shift",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/try...catch",
  "https://developer.mozilla.org/zh-CN/docs/Web/JavaScript/Reference/Global_Objects/Array",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Functions",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Error",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/slice",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/delete",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Functions",
  "https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control",
  "https://developer.mozilla.org/en-US/docs/Web/HTML/Element/video",
  "https://developer.mozilla.org/en-US/docs/Web/CSS/CSS_Colors/Color_picker_tool",
  "/fr/",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/flat",
  "https://developer.mozilla.org/zh-CN/docs/Web/JavaScript/Reference/Global_Objects/Object/assign",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Date/toLocaleDateString",
  "https://developer.mozilla.org/en-US/docs/Web/CSS/:not",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/switch",
  "https://developer.mozilla.org/en-US/docs/Learn/JavaScript/Objects/JSON",
  "https://developer.mozilla.org/en-US/docs/Web/HTTP/CSP",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Math/max",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/includes",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Loops_and_iteration",
  "https://developer.mozilla.org/en-US/docs/Web/HTTP/Cookies",
  "https://developer.mozilla.org/en-US/docs/Web/CSS/CSS_Animations/Using_CSS_animations",
  "https://developer.mozilla.org/en-US/docs/Web/HTML/Element/iframe",
  "https://developer.mozilla.org/zh-CN/docs/Web/JavaScript/Reference/Errors/Unexpected_type",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Logical_Operators",
  "https://developer.mozilla.org/en-US/docs/Web/Guide/Events/Creating_and_triggering_events",
  "https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/502",
  "https://developer.mozilla.org/en-US/docs/Mozilla/Mobile/Viewport_meta_tag",
  "https://developer.mozilla.org/zh-CN/docs/Web/JavaScript/Reference/Global_Objects/JSON/stringify",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Functions/Default_parameters",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/pop",
  "https://developer.mozilla.org/en-US/docs/Web/CSS/CSS_Grid_Layout",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide",
  "https://developer.mozilla.org/en-US/docs/Web/CSS/vertical-align",
  "https://developer.mozilla.org/en-US/docs/Web/CSS/transform-function/scale",
  "https://developer.mozilla.org/en-US/docs/Learn/Getting_started_with_the_web/HTML_basics",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Closures",
  "https://developer.mozilla.org/en-US/docs/Web/HTML/Element/button",
  "https://developer.mozilla.org/en-US/docs/Web/CSS/Using_CSS_custom_properties",
  "https://developer.mozilla.org/en-US/docs/Web/HTML/Element/input/tel",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Date/UTC",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Functions/arguments",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Date/toISOString",
  "https://developer.mozilla.org/en-US/docs/Learn/JavaScript/First_steps",
  "https://developer.mozilla.org/en-US/docs/Web/HTML/Element/a",
  "/ru/",
  "https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/500",
  "https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Complete_list_of_MIME_types",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/indexOf",
  "https://developer.mozilla.org/en-US/docs/Learn/Server-side/Express_Nodejs/mongoose",
  "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/await",
  "https://developer.mozilla.org/en-US/docs/Web/CSS/box-sizing"
]