Skip to content

Instantly share code, notes, and snippets.

View jbylund's full-sized avatar

Joseph Bylund jbylund

View GitHub Profile
Modern Merfolk:
4 Aether Vial
2 Cavern of Souls
1 Fiery Islet
3 Force of Negation
1 Glasspool Mimic
9 Island
4 Lord of Atlantis
4 Master of the Pearl Trident
4 Merfolk Trickster
import argparse
import base64
import csv
import gzip
import json
import multiprocessing
import sys
import tempfile
import logging
#!/usr/bin/env python
"""
let's try to do this without subprocess?
"""
# stdlib
import argparse
import datetime
import os
import shelve
import collections
import operator
import cProfile
import time
import os
from .providers import AbstractResolver
from .structs import DirectedGraph, IteratorMapping, build_iter_view
import datetime
import json
import logging
import re
import time
import xml.etree.ElementTree
import traceback
import dateutil.parser
import pg8000
@jbylund
jbylund / fetch_metadata.py
Created August 7, 2020 20:55
get aws metadata
#!/usr/bin/env python
import requests
import json
def crawl(iurl):
results = [iurl.rstrip("/") + "/" + x for x in requests.get(iurl).text.split("\n")]
for endpoint in (x for x in results if not x.endswith("/")):
ept = requests.get(endpoint).text
if ept and ept[0] in '{[':
ept = json.loads(ept)
PARSEPATTERN = re.compile(r"(?P<year>\d{4})-(?P<month>\d{2})-(?P<day>\d{2})T(?P<hour>\d{2}):(?P<minute>\d{2}):(?P<second>\d{2})\.000Z")
def _parse_timestamp_with_tzinfo(value, tzinfo):
"""Parse timestamp with pluggable tzinfo options."""
if isinstance(value, (str,)):
match = PARSEPATTERN.match(value)
if match is not None:
try:
return datetime.datetime(**{k: int(v) for k, v in match.groupdict().items()}).replace(tzinfo=tzinfo())
except ValueError:
pass
#!/usr/bin/python
import json
import time
import cfscrape
import bs4
def get_args():
pass
version: v1.0
name: Initial Pipeline
agent:
machine:
type: e1-standard-2
os_image: ubuntu1804
blocks:
- name: 'Block #1'
task:
prologue:
#!/usr/bin/env python
"""
put files to copy on
=> pipe 1 (contains files to copy)
copy from camera into tempdir on same filesystem as the import dir (one at a time)
=> pipe 2 (contains files on local fs)
move into correct locations and read metadata (in parallel)
=> pipe 3 (contains files & metadata)
write into sqlite database (one at a time)
=> pipe 4 finished