Unique server response headers accumulate across fetches to subsequent URLs

esp32spi_localtime_MODIFIED.py

"""
PyPortal CP 4.0.0-beta.3 + adafruit-circuitpython-bundle-4.x-mpy-20190307

Using esp32spi_localtime.py as a base, add in subsequent `wifi.get` calls to other websites.  
Note that the original file did not use `reponse.close()`.

Test:
GET server headers from several different websites sequentially; examine server headers. 
Between each fetch, call `response.close()` _and_ `response = None`

Result:
Server response bodies (text, json, etc.) are unique and correct for each website. 
Server headers with keys in common from fetch to fetch have their values changed accurately, 
*but* server headers with keys that are unique from fetch to fetch will accumulate. 
After fetching from several different servers, the returned set of accumulated headers can be quite large.

See Output below.

"""
import time
import board
import busio
from digitalio import DigitalInOut
import neopixel
from adafruit_esp32spi import adafruit_esp32spi
from adafruit_esp32spi import adafruit_esp32spi_wifimanager
import rtc

# Get wifi details and more from a secrets.py file
try:
    from secrets import secrets
except ImportError:
    print("WiFi secrets are kept in secrets.py, please add them there!")
    raise

print("ESP32 local time")

TIME_API = "http://worldtimeapi.org/api/ip"
CHEERLIGHTS_API = "https://api.thingspeak.com/channels/1417/feeds.json?results=1"
SIMPLETEST_TEXT_URL = "http://wifitest.adafruit.com/testwifi/index.html"
SIMPLETEST_JSON_URL = "http://api.coindesk.com/v1/bpi/currentprice/USD.json"

esp32_cs = DigitalInOut(board.ESP_CS)
esp32_ready = DigitalInOut(board.ESP_BUSY)
esp32_reset = DigitalInOut(board.ESP_RESET)
spi = busio.SPI(board.SCK, board.MOSI, board.MISO)
esp = adafruit_esp32spi.ESP_SPIcontrol(spi, esp32_cs, esp32_ready, esp32_reset)
"""Use below for Most Boards"""
status_light = neopixel.NeoPixel(board.NEOPIXEL, 1, brightness=0.2) # Uncomment for Most Boards
"""Uncomment below for ItsyBitsy M4"""
#status_light = dotstar.DotStar(board.APA102_SCK, board.APA102_MOSI, 1, brightness=0.2)
wifi = adafruit_esp32spi_wifimanager.ESPSPI_WiFiManager(esp, secrets, status_light)

the_rtc = rtc.RTC()

response = None
while True:
    try:
        print("Fetching json from", TIME_API)
        response = wifi.get(TIME_API)
        break
    except (ValueError, RuntimeError) as e:
        print("Failed to get data, retrying\n", e)
        continue

json = response.json()
current_time = json['datetime']
the_date, the_time = current_time.split('T')
year, month, mday = [int(x) for x in the_date.split('-')]
the_time = the_time.split('.')[0]
hours, minutes, seconds = [int(x) for x in the_time.split(':')]

# We can also fill in these extra nice things
year_day = json['day_of_year']
week_day = json['day_of_week']
is_dst = json['dst']

now = time.struct_time((year, month, mday, hours, minutes, seconds, week_day, year_day, is_dst))
print(now)
the_rtc.datetime = now

response.close()
print()

response = None
while True:
    try:
        print("Fetching json from", TIME_API)
        response = wifi.get(TIME_API)
        break
    except (ValueError, RuntimeError) as e:
        print("Failed to get data, retrying\n", e)
        continue
print(response.headers)
response.close()
print()

response = None
while True:
    try:
        print("Fetching json from", CHEERLIGHTS_API)
        response = wifi.get(CHEERLIGHTS_API)
        
        break
    except (ValueError, RuntimeError) as e:
        print("Failed to get data, retrying\n", e)
        continue
print(response.headers)
response.close()
print()

response = None
while True:
    try:
        print("Fetching text from", SIMPLETEST_TEXT_URL)
        response = wifi.get(SIMPLETEST_TEXT_URL)
        break
    except (ValueError, RuntimeError) as e:
        print("Failed to get data, retrying\n", e)
        continue
print(response.headers)
response.close()
print()

response = None
while True:
    try:
        print("Fetching json from", SIMPLETEST_JSON_URL)
        response = wifi.get(SIMPLETEST_JSON_URL)
        break
    except (ValueError, RuntimeError) as e:
        print("Failed to get data, retrying\n", e)
        continue
print(response.headers)
response.close()
print()

response = None
while True:
    try:
        print("Fetching json from", TIME_API)
        response = wifi.get(TIME_API)
        break
    except (ValueError, RuntimeError) as e:
        print("Failed to get data, retrying\n", e)
        continue
print(response.headers)
response.close()
print()

while True:
    print(time.localtime())
    time.sleep(60)
"""
Output: 

Auto-reload is on. Simply save files over USB to run them or enter REPL to disable.
main.py output:
ESP32 local time
Fetching json from http://worldtimeapi.org/api/ip
struct_time(tm_year=2019, tm_mon=3, tm_mday=8, tm_hour=17, tm_min=53, tm_sec=30, tm_wday=5, tm_yday=67, tm_isdst=False)

Fetching json from http://worldtimeapi.org/api/ip
{'server': 'Cowboy', 'content-type': 'application/json; charset=utf-8', 'date': 'Fri, 08 Mar 2019 23:53:30 GMT', 'cache-control': 'max-age=0, private, must-revalidate', 'connection': 'close', 'content-length': '240', 'via': '1.1 vegur'}

Fetching json from https://api.thingspeak.com/channels/1417/feeds.json?results=1
{'date': 'Fri, 08 Mar 2019 23:53:34 GMT', 'content-type': 'application/json; charset=utf-8', 'x-runtime': '0.008949', 'x-powered-by': 'Phusion Passenger 4.0.57', 'x-request-id': '1ba5955d-3fc3-4e43-b683-fbc1690f0047', 'server': 'nginx/1.9.3 + Phusion Passenger 4.0.57', 'connection': 'close', 'access-control-allow-methods': 'GET, POST, PUT, OPTIONS, DELETE, PATCH', 'access-control-allow-headers': 'origin, content-type, X-Requested-With', 'content-length': '240', 'etag': '"66d826d4a37afc35b69723dea81eec07"', 'status': '200 OK', 'x-frame-options': 'SAMEORIGIN', 'via': '1.1 vegur', 'access-control-allow-origin': '*', 'cache-control': 'max-age=0, private, must-revalidate', 'access-control-max-age': '1800'}

Fetching text from http://wifitest.adafruit.com/testwifi/index.html
{'via': '1.1 vegur', 'content-length': '73', 'last-modified': 'Thu, 16 Feb 2017 17:42:29 GMT', 'accept-ranges': 'bytes', 'x-powered-by': 'Phusion Passenger 4.0.57', 'access-control-allow-origin': '*', 'access-control-max-age': '1800', 'server': 'nginx/1.10.3 (Ubuntu)', 'etag': '"58a5e485-49"', 'x-frame-options': 'SAMEORIGIN', 'connection': 'close', 'content-type': 'text/html', 'x-runtime': '0.008949', 'date': 'Fri, 08 Mar 2019 23:53:35 GMT', 'x-request-id': '1ba5955d-3fc3-4e43-b683-fbc1690f0047', 'cache-control': 'max-age=0, private, must-revalidate', 'access-control-allow-methods': 'GET, POST, PUT, OPTIONS, DELETE, PATCH', 'access-control-allow-headers': 'origin, content-type, X-Requested-With', 'status': '200 OK'}

Fetching json from http://api.coindesk.com/v1/bpi/currentprice/USD.json
{'via': '1.1 f6899c291987e0abbc2941962c0c7fb5.cloudfront.net (CloudFront)', 'content-length': '407', 'last-modified': 'Thu, 16 Feb 2017 17:42:29 GMT', 'accept-ranges': 'bytes', 'expires': 'Fri, 08 Mar 2019 23:54:07 UTC', 'x-powered-by': 'Fat-Free Framework', 'access-control-allow-origin': '*', 'access-control-max-age': '1800', 'server': 'nginx/1.14.1', 'etag': '"58a5e485-49"', 'age': '7', 'x-frame-options': 'SAMEORIGIN', 'x-cache': 'Hit from cloudfront', 'connection': 'close', 'x-amz-cf-id': 'XVRJdnWizWiYWE8dQhAMbqqxnAl047beYiJg47qsfUYf1zXa-l3TDw==', 'content-type': 'application/javascript', 'x-runtime': '0.008949', 'date': 'Fri, 08 Mar 2019 23:53:29 GMT', 'x-request-id': '1ba5955d-3fc3-4e43-b683-fbc1690f0047', 'cache-control': 'max-age=15', 'access-control-allow-methods': 'GET, POST, PUT, OPTIONS, DELETE, PATCH', 'access-control-allow-headers': 'origin, content-type, X-Requested-With', 'status': '200 OK'}

Fetching json from http://worldtimeapi.org/api/ip
{'via': '1.1 vegur', 'content-length': '240', 'last-modified': 'Thu, 16 Feb 2017 17:42:29 GMT', 'accept-ranges': 'bytes', 'expires': 'Fri, 08 Mar 2019 23:54:07 UTC', 'x-powered-by': 'Fat-Free Framework', 'access-control-allow-origin': '*', 'access-control-max-age': '1800', 'server': 'Cowboy', 'etag': '"58a5e485-49"', 'age': '7', 'x-frame-options': 'SAMEORIGIN', 'x-cache': 'Hit from cloudfront', 'connection': 'close', 'x-amz-cf-id': 'XVRJdnWizWiYWE8dQhAMbqqxnAl047beYiJg47qsfUYf1zXa-l3TDw==', 'content-type': 'application/json; charset=utf-8', 'x-runtime': '0.008949', 'date': 'Fri, 08 Mar 2019 23:53:36 GMT', 'x-request-id': '1ba5955d-3fc3-4e43-b683-fbc1690f0047', 'cache-control': 'max-age=0, private, must-revalidate', 'access-control-allow-methods': 'GET, POST, PUT, OPTIONS, DELETE, PATCH', 'access-control-allow-headers': 'origin, content-type, X-Requested-With', 'status': '200 OK'}

struct_time(tm_year=2019, tm_mon=3, tm_mday=8, tm_hour=17, tm_min=53, tm_sec=36, tm_wday=4, tm_yday=67, tm_isdst=-1)
"""

Closed: adafruit/Adafruit_CircuitPython_ESP32SPI#22