Skip to content

Instantly share code, notes, and snippets.

View RusselWebber's full-sized avatar

Russel Webber RusselWebber

View GitHub Profile
@RusselWebber
RusselWebber / persistent_cache.py
Created February 15, 2024 11:25
Cache a pandas dataframe locally so the data is not fetched every time the sheet is opened
import logging
import os
import pandas as pd
LOG = logging.getLogger(__name__)
def read_dataframe_from_pickle_or_url(url: str, path: str) -> pd.DataFrame:
"""Read JSON data from the URL and convert to a pandas dataframe.
The dataframe is cached locally at the supplied path.
def hello_gist(name):
return f"Hello {name}, this code was imported from a gist."
class SharedResourceClass:
"""This could be any resource that you would like to share.
Perhaps because the class is slow to initialise or you want to track usage
"""
def __init__(self, initial_value: int) -> None:
self._shared_resource = initial_value
def access_shared_resource(self) -> int:
from typing import List, Any
def only_strings(n):
result = []
for i in range(int(n)):
result.append(["col1", "col2"])
return result
import asyncio
import numpy as np
from io import BytesIO
from contextlib import closing
from aiokafka import AIOKafkaConsumer
async def kafka_topic_subscription(topic: str, server: str, fps: str) -> np.ndarray:
"""Subscribe to a numpy array from a Kafka topic"""
consumer = AIOKafkaConsumer(
topic, bootstrap_servers=server, auto_offset_reset="earliest"
import asyncio
async def async_counter(stop: int, updates_per_second: int) -> int:
"""Asynchronously generates a sequence of numbers up to stop at a rate of updates_per_second."""
for i in range(stop):
await asyncio.sleep(1.0 / updates_per_second)
yield i
import time
def counter(stop: int, updates_per_second: int) -> int:
"""Generates a sequence of numbers up to stop at a rate of updates_per_second."""
for i in range(stop):
time.sleep(1.0 / updates_per_second)
yield i