Skip to content

Instantly share code, notes, and snippets.

View zzstoatzz's full-sized avatar
⚡

nate nowack zzstoatzz

⚡
52 followers · 6 following
View GitHub Profile
@zzstoatzz
zzstoatzz / gist:197bfd744d172174dd27e4fc7f91a05c
Last active June 4, 2020 01:35
Create Fourier Dash
##### Frequency identification
# imports
from IPython.display import HTML
from FREQhelpers import createDash
import sys, time
# animation / dynamics inputs
resolution = .15
duration, T = 30, 2 # seconds, seconds/cycle
freqs, signal = [5, 10], False
@zzstoatzz
zzstoatzz / spec.json
Created April 26, 2021 00:56
airbyte-spec
{
"documentationUrl": "https://docs.airbyte.io",
"connectionSpecification": {
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "Smartsheets Source Spec",
"type": "object",
"required": ["access_token", "spreadsheet_id"],
"additionalProperties": false,
"properties": {
"access_token": {
@zzstoatzz
zzstoatzz / source.py
Created April 26, 2021 01:22
airbyte source connector pseudo class definition
# main class definition
class SourceSmartsheets(Source):
def check(self, logger: AirbyteLogger, config: json) -> AirbyteConnectionStatus:
# implement "check" that your credentials allow a good connection to your data source
# e.g. Smartsheets api call on my smartsheet of interest -> status 200
return AirbyteConnectionStatus(status=Status.FAILED)
def discover(self, logger: AirbyteLogger, config: json) -> AirbyteCatalog:
# create a JSONschema catalog for your source based on column names and types
@zzstoatzz
zzstoatzz / pgn-etl-flow.py
Created March 7, 2022 01:27
from chessdotcom import get_player_game_archives
from io import StringIO
from prefect import flow, task
from prefect.deployments import DeploymentSpec
from prefect.tasks import task_input_hash
from typing import List, Tuple
import boto3, chess.pgn as pgn, pandas as pd, requests
# https://python-chess.readthedocs.io/en/latest/pgn.html
class Game(pgn.Game):
@zzstoatzz
zzstoatzz / games.csv
Created March 28, 2022 03:29
Event Site Date Round White Black Result BlackElo CurrentPosition ECO ECOUrl EndDate EndTime Link StartTime Termination TimeControl Timezone UTCDate UTCTime WhiteElo pgn
Live Chess Chess.com 2021.12.03 - glen_bishop n80n8 1-0 414 4Rk2/ppp3p1/7p/5pN1/2n5/2b5/P4PPP/4R1K1 b - - C51 https://www.chess.com/openings/Giuoco-Piano-Game-Evans-Accepted-McDonnell-Defense-6.d4 2021.12.03 18:41:27 https://www.chess.com/game/live/32295537927 18:32:34 glen_bishop won by checkmate 600 UTC 2021.12.03 18:32:34 1226 1. e4 { [%clk 0:10:00] } 1... e5 { [%clk 0:10:00] } 2. Nf3 { [%clk 0:09:55.7] } 2... Nc6 { [%clk 0:09:54.8] } 3. Bc4 { [%clk 0:09:48] } 3... Bc5 { [%clk 0:09:35.6] } 4. b4 { [%clk 0:09:43.8] } 4... Bxb4 { [%clk 0:09:28.3] } 5. c3 { [%clk 0:09:35.5] } 5... Bc5 { [%clk 0:09:25.8] } 6. d4 { [%clk 0:09:33.2] } 6... Bb6 { [%clk 0:09:13.2] } 7. dxe5 { [%clk 0:09:07.2] } 7... Nge7 { [%clk 0:09:01] } 8. O-O { [%clk 0:08:56.7] } 8... d5 { [%clk 0:08:54.2] } 9. exd5 { [%clk 0:08:49.4] } 9... Na5 { [%clk 0:08:43.5] } 10. Bb5+ {
@zzstoatzz
zzstoatzz / logs_no_caching.log
Last active March 28, 2022 20:54
❯ python flow.py
10:56:13.218 | INFO | prefect.engine - Created flow run 'marvellous-serval' for flow "Store chess.com users' games"
10:56:13.218 | INFO | Flow run 'marvellous-serval' - Using task runner 'ConcurrentTaskRunner'
Checking for games on chess.com from: n80n8
10:56:13.522 | INFO | Flow run 'marvellous-serval' - Created task run 'get_games-d2192cf5-0' for task 'get_games'
10:56:13.558 | INFO | Flow run 'marvellous-serval' - Created task run 'get_games-d2192cf5-1' for task 'get_games'
GET https://api.chess.com/pub/player/n80n8/games/2021/11
/Users/nate/opt/miniconda3/envs/chess/lib/python3.7/site-packages/prefect/client.py:1196: UserWarning: No default storage has been set on the server. Using temporary local storage for results.
"No default storage has been set on the server. "
@zzstoatzz
zzstoatzz / logs_with_caching.log
Created March 28, 2022 20:54
❯ python flow.py
15:53:15.919 | INFO | prefect.engine - Created flow run 'shapeless-partridge' for flow "Store chess.com users' games"
15:53:15.919 | INFO | Flow run 'shapeless-partridge' - Using task runner 'ConcurrentTaskRunner'
Checking for games on chess.com from: n80n8
15:53:16.218 | INFO | Flow run 'shapeless-partridge' - Created task run 'get_games-d2192cf5-0' for task 'get_games'
15:53:16.254 | INFO | Flow run 'shapeless-partridge' - Created task run 'get_games-d2192cf5-1' for task 'get_games'
15:53:16.281 | INFO | Task run 'get_games-d2192cf5-0' - Finished in state Cached(None, type=COMPLETED)
15:53:16.287 | INFO | Flow run 'shapeless-partridge' - Created task run 'get_games-d2192cf5-2' for task 'get_games'
15:53:16.320 | INFO | Task run 'get_games-d2192cf5-1' - Finished in state Cached(None, type=COMPLETED)
@zzstoatzz
zzstoatzz / airbyte_sync_logs.txt
Created April 28, 2022 19:05
❯ python airbyte_syncs.py
03:46:03 | prefect.engine - Created flow run 'thick-seahorse' for flow 'example_trigger_sync_flow'
03:46:03 | Flow run 'thick-seahorse' - Using task runner 'ConcurrentTaskRunner'
03:46:03 | Flow run 'thick-seahorse' - Created task run 'trigger_sync-35f0e9c2-0' for task 'trigger_sync'
03:46:03 | prefect - trigger airbyte connection: e1b2078f-882a-4f50-9942-cfe34b2d825b, poll interval 3 seconds
03:46:03 | prefect - pending
03:46:06 | prefect - running
03:46:09 | prefect - running
03:46:12 | prefect - running
03:46:16 | prefect - running
@zzstoatzz
zzstoatzz / airbyte_syncs.py
Last active April 28, 2022 19:08
from prefect import flow
from prefect_airbyte.connections import trigger_sync
@flow
def example_trigger_sync_flow():
# Run other tasks and subflows here
trigger_sync(
@zzstoatzz
zzstoatzz / export_configuration.py
Created April 28, 2022 19:44
import gzip
from prefect import flow, task
from prefect_airbyte.configuration import export_configuration
@task
def zip_and_write_somewhere(
airbyte_config: bytearray,
somewhere: str,
):