Skip to content

Instantly share code, notes, and snippets.

View zeryx's full-sized avatar

zeryx zeryx

View GitHub Profile
@zeryx
zeryx / algo_recursion.py
Last active November 26, 2019 21:02
a non-trival algorithm that recursively calls itself when it recieves a large batch of work requests.
import numpy as np
import tensorflow as tf
from multiprocessing import Manager, Pool
import Algorithmia
import os
import re
# This is code for most tensorflow image classification algorithms
# In this example we look at solving batch processing problems with algorithm recursion and pipelining.
@zeryx
zeryx / attention_nework_example.py
Last active November 20, 2019 22:29
using pytorch and attention layers to forecast timeseries
import torch
from torch import nn
import numpy as np
class GaussianNoise:
def __init__(self, stddev: float):
super(GaussianNoise, self).__init__()
self.stddev = stddev
import scala.reflect.runtime.universe._
import play.api.libs.json._
import scala.util.{Failure, Success, Try}
object AbstractAlgorithm {
trait AbstractAlgorithm[I, O] {
def apply(input: I): Try[O]
package com.algorithmia.handler
import play.api.libs.json._
import scala.util.{Failure, Success, Try}
trait AbstractAlgorithm[I <: AnyVal, O <: AnyVal] {
def apply(input:I): Try[O]
def load(): Try[Unit] = Success(())
implicit def inputReader: Reads[I] = Try(implicitly[Reads[I]]) match {
case Failure(_) => Json.reads[I]
package com.algorithmia.handler
import java.io.FileOutputStream
import java.io.PrintStream
import scala.util.Try
case class ResponseHandler(){
val FIFOPATH = "/tmp/algoout"
@zeryx
zeryx / export_deps.py
Last active October 11, 2019 18:57
shim files to enable docker copy of dynamic files
import os
import sys
import json
from shutil import copy
from uuid import uuid4
def load_filenames_file(path):
with open(path) as f:
files = set(f.readlines())
return files
FROM SOMEIMAGENAME as runner_intermediate
# We want to cache the following permanently, as these won't change frequently and take a very long time to move around and rebuild.
COPY --from=builder0 --chown=2222:2222 /some/artifact_dir /some/artifact_dir
RUN find /some/artifact_dir > /tmp/artifact_files_BEFORE
FROM SOMEOTHERIMAGE as builder
#This operation occurs on every build operation, therefore to increase perf we want all ops after this to be fast.
@zeryx
zeryx / create_headless.py
Created September 24, 2019 18:03
takes a pytorch resnet 18 model, strips off the last module and checks to make sure serialization still works properly.
import torch as th
from torchvision import models, transforms
from PIL import Image
import numpy as np
resnet18 = models.resnet18(pretrained=True)
modules=list(resnet18.children())[:-1]
model = th.nn.Sequential(*modules)
import torch as th
from torchvision import models, transforms
from PIL import Image
import numpy as np
model = models.resnet18(pretrained=True)
class Identity(th.nn.Module):
def __init__(self):
super(Identity, self).__init__()
$ cat algorithmia-r.Rcheck/00check.log
* using log directory ‘/home/zeryx/code/work/algorithmia-r.Rcheck’
* using R version 3.4.4 (2018-03-15)
* using platform: x86_64-pc-linux-gnu (64-bit)
* using session charset: UTF-8
* using option ‘--as-cran’
* checking for file ‘algorithmia-r/DESCRIPTION’ ... OK
* checking extension type ... Package
* this is package ‘algorithmia’ version ‘0.2.0’
* package encoding: UTF-8