evandiewald

from dotenv import load_dotenv
from sshtunnel import SSHTunnelForwarder
from sqlalchemy.engine import create_engine, Engine
from sqlalchemy.ext.asyncio import create_async_engine, AsyncEngine
from typing import Union
import os
from sqlalchemy.pool import NullPool


load_dotenv()

evandiewald

def layout(place=None):
    if place:
        global G, G2
        G, _, edges = load_map(place)
        G2 = nx.Graph(G)

        polyline = dl.Polyline(id="route-line", positions=[], color="red", weight=10, fillColor="red", fillOpacity=1.0)
        patterns = [dict(repeat='100',
                         arrowHead=dict(pixelSize=15, polygon=False, pathOptions=dict(color="red", stroke=True)),
                         line=dict(pixelSize=10, pathOptions=dict(color='#f00', weight=20)))]

evandiewald

def find_best_loop(G: nx.Graph, root, target_dist, tol=1.0, min_elev: Optional[Union[int, float]] = None,
                   dataset: Optional[rasterio.DatasetReader] = None):
    if min_elev and not dataset:
        raise ValueError("If asking for elevation data, you must include a rasterio dataset")
    error = 1e8
    best_path = []
    for n in G.nodes():
        if nx.has_path(G, root, n):
            shortest_path = nx.shortest_path(G, root, n)
            paths = nx.all_simple_paths(G, root, n, cutoff=10)

evandiewald

@callback(
    Output("route-path", "data"),
    Input("map", "click_lat_lng"),
    Input("distance-target", "value"),
    Input("elevation-min", "value"),
    Input("mode-select", "value"),
    State("route-path", "data")
)
def cycles(click_lat_lng, distance_target: Optional[Union[int, float]], elevation_min: Optional[Union[int, float]], mode, path):
    if click_lat_lng is None:

evandiewald

def get_elevation_profile_of_segment(dataset: rasterio.DatasetReader, coords: list[list]):
    """
    Get the elevation profile (distance vs. altitude) of a path segment from the list of coordinates.
    Args:
        dataset: The opened rasterio dataset for the SRTM global topography data. 
        coords: The path coordinates in [[lon1, lat1], [lon2, lat2], ...] format.

    Returns: The distance (in miles) and elevation (in feet) vectors.
    """
    # coordinates are [lon, lat], flip for rasterio

evandiewald

def load_map(place: str, graph_only: bool = False) -> (nx.MultiDiGraph, dict):
    """
    Load OSM trail data for a given region. Initially check if the graph has already been cached on disk, otherwise it will be downloaded.
    Args:
        place: The geocode of the region of interest, e.g. 'Shenandoah National Park, Virginia, USA'
        graph_only: If true, return only the NetworkX graph, not the geojson.

    Returns: The dataset as a NetworkX MultiGraph, the nodes geojson, the edges geojson

    """

evandiewald

with hashes as 

(select transaction_hash, actor from transaction_actors where 
actor = {{address}} 
and actor_role = 'witness' 
and block > (select max(height) - {{n_blocks}} from blocks)),

target_transactions as 
(select fields, hash from transactions where 
(type = 'poc_receipts_v2' or type = 'poc_receipts_v1') 

evandiewald

Build a Lightweight Helium ETL with blockchain-node

Want to run your own analyses on Helium blockchain data, without the expense and complexity of a operating a full node? In this article, we'll explore blockchain-node, a lightweight block follower that can be leveraged for maintaining real-time datasets. In Part I, we'll introduce the benefits (and limitations) of this tool, hardware requirements, and usage patterns. Then, we'll walk through the core components of an ETL service that inserts transactions into an analytics-friendly SQL database.

Part I: What is blockchain-node?

From the blockchain-node README:

This is an Erlang application that is a Helium Blockchain node. It follows the blockchain and exposes functionality using a JSONRPC 2.0 API.

evandiewald

import requests
import pandas as pd


ACCOUNT = "B58_WALLET_ADDRESS"

# UTC timestamps are given in ISO 8601 format
MIN_TIME = "2022-02-01"
MAX_TIME = "2022-03-01"

evandiewald

DeWi Grant Bounty: Analytics-Friendly Data Dumps

Description

Open-source, easily-accessible datasets have been shown to spurn innovation in fields like machine learning and data visualization. While the DeWi Metabase platform is suitable for ad hoc analyses and dashboards, queries can be slow and rate-limited. On the other hand, blockchain-etl allows for direct programmatic access to the entire ledger, but extensive hardware requirements and the lengthy initial sync time present a significant barrier to entry.

This grant serves a gap in Helium’s existing data engineering infrastructure by producing periodic static data dumps of key tables over discrete time periods.

The offering will accelerate exploratory analyses related to anti-gaming, token flow, coverage mapping, data usage, and more by enabling data scientists to interact with manageable datasets locally, in the programming language of their choice.

Deliverables