joshlk

Steps to debug a program on a remote machine without using remote deployment.

1. Start the debug server in pycharm and specify a port such as 21000
2. SSH remote forward a port e.g. ssh host -R 21000:localhost:21000
3. Start a Python process and insert the following line (first pip install pydevd-pycharm`):

import pydevd_pycharm; pydevd_pycharm.settrace('localhost', port=21000, stdoutToServer=True, stderrToServer=True)

joshlk

#! /bin/sh
set -ef

if test -n "$KSH_VERSION"; then
	puts() {
		print -r -- "$*"
	}
else
	puts() {
		printf '%s\n' "$*"

joshlk

joshlk

import heapq

class PriorityQueue:
    """
    A PriorityQueue or MinHeap implementation.
    Items with the smallest cost are popped first.
    """

    def __init__(self):
        self.h = []

joshlk

import numpy as np
from itertools import count, product

class DirectedAgglomerativeClustering:
    """
    Similar to `sklearn.cluster.AgglomerativeClustering` but the label is the root node. The root node is the root of a
    connected DAG (Directed Acyclic Graph).
    Also can frame algorithm as determining the weakly connected components and identifying the root.
    Algorithm is naive implementation and O(n^3)
    """

joshlk

from libc.stdlib cimport rand, srand, RAND_MAX
from libc.time cimport time

def get_RAND_MAX():
    return RAND_MAX

"""
unsigned long is at least 32 bits and so in the range [−2,147,483,647, +2,147,483,647]
"""

joshlk

import trio
import tqdm

class TrioProgress(trio.abc.Instrument):

    def __init__(self, total, notebook_mode=False, **kwargs):
        if notebook_mode:
            from tqdm import tqdm_notebook as tqdm
        else:
            from tqdm import tqdm

joshlk

from pyspark import SparkContext
from pyspark.sql import SQLContext
import numpy as np

sc = SparkContext()
sqlContext = SQLContext(sc)

# Create dummy pySpark DataFrame with 1e5 rows and 16 partitions
df = sqlContext.range(0, int(1e5), numPartitions=16)

joshlk

joshlk

# SSH port forwarding/tunnel
# If I want to be able to access port 6006 on a remote server locally
ssh -L 6006:localhost:6006 user@remote