gordonje

VAN Survey Question Responses

This script will:

• Send an authenticated request to the /surveyQuestions endpoint of VAN's API
• Unnest responses from each item
• Output to a local file in newline delimited JSON (aka JSONL, which is...not?...the same as NDJSON...)

Dependencies

gordonje

Editor Input	Preview Output
1	1
=(2 + 2)	=(2 + 2)
=(51 / 3)	=(51 / 3)
=(B1 + B3)	=(B1 + B3)
=(TOTAL)	=(TOTAL)

gordonje

import requests
from bs4 import BeautifulSoup
import json

url_stub = "https://results.mo.gov"

workbook_url = f"{url_stub}/t/COVID19/views/VaccinationsDashboard/Vaccinations"

workbook_params = {
    ':embed': 'y',

gordonje

import requests
from time import sleep
from multiprocessing import Pool

session = None

def set_global_session():
    global session
    if not session:
        session = requests.Session()

gordonje

import requests
from time import sleep


session = requests.Session()

def cache_page(identifier):
    sleep(3)
    url = f'https://mycourts.in.gov/PORP/Search/Detail?ID={identifier}'
    r = session.get(url)

gordonje

youtube-dl --write-info-json --all-subs --write-all-thumbnails \
  -o '~/Desktop/warrenmayer-best-merge/%(title)s-%(id)s/%(title)s-%(id)s.%(ext)s' \
  https://www.youtube.com/user/warrenmayer/

gordonje

youtube-dl --write-info-json --all-subs --write-all-thumbnails \
  -f 'bestvideo[ext=mp4]+bestaudio[ext=m4a]' \
  -o '~/Desktop/warrenmayer-best-mp4/%(title)s-%(id)s/%(title)s-%(id)s.%(ext)s' \
  https://www.youtube.com/user/warrenmayer/

gordonje

youtube-dl --write-info-json --all-subs --write-all-thumbnails -f best \
  -o '~/Desktop/warrenmayer-best-single/%(title)s-%(id)s/%(title)s-%(id)s.%(ext)s' \
  https://www.youtube.com/user/warrenmayer/

gordonje

select link_group_id, link_type_description, count(*)
from calaccess_processed_candidate_committees a
join (
    select cand_filer_id, committee_filer_id, count(*)
    from calaccess_processed_candidate_committees
    group by 1, 2
    having count(*) > 1
) as dupes
on a.cand_filer_id = dupes.cand_filer_id
and a.committee_filer_id = dupes.committee_filer_id

gordonje

CREATE TABLE calaccess_processed_candidate_committees AS
SELECT   
        @link_type as link_group_id,
        lu."CODE_DESC" AS link_type_description,
        cand_filer_id,
        committee_filer_id,
        MIN(session) AS first_session,
        MAX(session) AS last_session,
        MIN(effective_date) AS first_effective_date,
        MAX(effective_date) AS last_effective_date,