Skip to content

Instantly share code, notes, and snippets.

@quasiben
quasiben / gist:3737828
Created September 17, 2012 14:52
sys.path From Map
['/usr/local/var/disco/data/localhost/ce/Job@543:cf5b5:7dfe1/home/quasiben/Downloads/epd_free-7.3-2-rh5-x86_64/lib/python2.7/site-packages/disco/worker/classic', '/usr/local/lib/python2.7/dist-packages/pyasn1-0.0.13b-py2.7.egg', '/usr/local/lib/python2.7/dist-packages/decorator-3.3.1-py2.7.egg', '/usr/local/lib/python2.7/dist-packages/workerpool-0.9.2-py2.7.egg', '/usr/local/lib/python2.7/dist-packages/boto-2.3.0-py2.7.egg', '/usr/local/lib/python2.7/dist-packages/ssh-1.7.13-py2.7.egg', '/usr/local/var/disco/data/localhost/ce/Job@543:cf5b5:7dfe1/lib', '/usr/local/var/disco/data/localhost/ce/Job@543:cf5b5:7dfe1/home/quasiben/Research/DiscoLogProcessing', '/usr/local/var/disco/data/localhost/ce/Job@543:cf5b5:7dfe1/home/quasiben/Downloads/epd_free-7.3-2-rh5-x86_64/lib/python2.7/site-packages/Cython-0.16-py2.7-linux-x86_64.egg', '/usr/local/var/disco/data/localhost/ce/Job@543:cf5b5:7dfe1/home/quasiben/Downloads/epd_free-7.3-2-rh5-x86_64/lib/python2.7/site-packages/Markdown-2.2.0-py2.7.egg', '/usr/local/var/disco/
@quasiben
quasiben / numba-diffusion.py
Created September 26, 2012 13:37
Diffusion with Sink using Python and Numba
import numpy as np
from numba.decorators import jit
from numba import *
mu = 0.1
Lx, Ly = 101, 101
@jit(arg_types=[double[:,:],double[:,:],int32])
def diffusionObstacleStep(u,tempU,iterNum):
for n in range(iterNum):
@quasiben
quasiben / cmagic.py
Last active December 17, 2015 16:19
# -*- coding: utf-8 -*-
"""C magic.
This is a very simple magic for compiling and importing C code with ctypes.
This derives from biteymagic.py, by Bradley Froehle:
https://gist.github.com/bfroehle/3458310
You must provide the function names you would like exported.
import numpy as np
from bokeh.plotting import *
output_notebook()
def dscatter(color="black"):
def decorator(fn):
x,y = fn()
scatter(x,y, color=color, tools="pan,wheel_zoom,box_zoom,reset,resize")
show()
return decorator
@quasiben
quasiben / python_postgres_frankenlanguage.py.sql
Created June 30, 2014 18:19
python_postgres_frankenlanguage
CREATE OR REPLACE FUNCTION bz_select_html(param_search text)
RETURNS SETOF text AS
$$
from lxml import html
import requests
page = requests.get('http://econpy.pythonanywhere.com/ex/001.html')
param = param_search.lower()
tree = html.fromstring(page.text)
@quasiben
quasiben / anaconda_spark.py
Created July 3, 2014 18:39
Anaconda Spark AMI Error
>>> sc.appName
u'PySparkShell'
>>> sc._conf.getAll()
[(u'spark.executor.extraLibraryPath', u'/root/ephemeral-hdfs/lib/native/'), (u'spark.executor.memory', u'6154m'), (u'spark.submit.pyFiles', u''), (u'spark.app.name', u'
PySparkShell'), (u'spark.executor.extraClassPath', u'/root/ephemeral-hdfs/conf'), (u'spark.master', u'spark://XX-XXX-XXX-XXX.compute-1.amazonaws.com:7077')]
>>> file = sc.textFile("hdfs://XX-XXX-XXX-XXX.amazonaws.com:9000/user/root/chekhov.txt")
14/07/03 18:24:02 INFO storage.MemoryStore: ensureFreeSpace(34388) called with curMem=0, maxMem=309225062
14/07/03 18:24:02 INFO storage.MemoryStore: Block broadcast_0 stored as values in memory (estimated size 33.6 KB, free 294.9 MB)
>>> file.take(2)
14/07/03 18:24:08 WARN snappy.LoadSnappy: Snappy native library is available
@quasiben
quasiben / solr_schema.xml
Created September 19, 2014 14:59
schema for solr 4.9.0 used with nutch 1.9
<?xml version="1.0" encoding="UTF-8" ?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
SLF4J: Class path contains multiple SLF4J bindings.
SLF4J: Found binding in [jar:file:/Users/quasiben/Research/ContinuumDev/Memex/nutch_application/nutch/runtime/local/lib/slf4j-log4j12-1.6.1.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: Found binding in [jar:file:/Users/quasiben/anaconda/envs/nutchpy/lib/python2.7/site-packages/nutchpy/java_libs/seqreader-app-1.0-SNAPSHOT-jar-with-dependencies.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation.
SLF4J: Actual binding is of type [org.slf4j.impl.Log4jLoggerFactory]
2014-09-29 16:31:55.463 java[17872:5403] Unable to load realm info from SCDynamicStore
URL: /var/folders/1t/t94brwgx7sjcn8jgz4gr3_c00000gq/T/tmpaJeuZN
14/09/29 16:31:56 INFO crawl.Injector: Injector: starting at 2014-09-29 16:31:56
14/09/29 16:31:56 INFO crawl.Injector: Injector: crawlDb: /Users/quasiben/Research/ContinuumDev/Memex/nutchpy/crawl
14/09/29 16:31:56 INFO crawl.Injector: Injector: urlDir: /var/folders/1t/t9
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
@quasiben
quasiben / Vagrantfile
Created November 11, 2015 21:24
Vagrant file with one private network
cat Vagrantfile
# -*- mode: ruby -*-
# vi: set ft=ruby :
VAGRANTFILE_API_VERSION = "2"
Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
config.vm.box = "1vq9/trusty64"
config.vm.synced_folder ".", "/vagrant", disabled: true
config.vm.define :dev do |box|
box.vm.hostname = "dev"
box.vm.network "private_network", ip: "192.168.222.100"