| """This file extracts faster-rcnn features and bounding box coordinates""" | |
| import pdb | |
| import argparse | |
| import numpy as np | |
| import tensorflow as tf | |
| import PIL.Image as PILI | |
| def session(sess, feat_conv, feat_avg, boxes, classes, scores, image_tensor, image): | |
| feat_conv_out, feat_avg_out, boxes_out, classes_out, scores_out = sess.run([ |
| from bs4 import BeautifulSoup | |
| import requests | |
| import re | |
| import urllib2 | |
| import os | |
| import argparse | |
| import sys | |
| import json | |
| # adapted from http://stackoverflow.com/questions/20716842/python-download-images-from-google-image-search |
Disclaimer: This piece is written anonymously. The names of a few particular companies are mentioned, but as common examples only.
This is a short write-up on things that I wish I'd known and considered before joining a private company (aka startup, aka unicorn in some cases). I'm not trying to make the case that you should never join a private company, but the power imbalance between founder and employee is extreme, and that potential candidates would
| import avro.schema | |
| import io, random | |
| from avro.io import DatumWriter, DatumReader | |
| import avro.io | |
| # Path to user.avsc avro schema | |
| schema_path="user.avsc" | |
| schema = avro.schema.parse(open(schema_path).read()) | |
RDBMS-based job queues have been criticized recently for being unable to handle heavy loads. And they deserve it, to some extent, because the queries used to safely lock a job have been pretty hairy. SELECT FOR UPDATE followed by an UPDATE works fine at first, but then you add more workers, and each is trying to SELECT FOR UPDATE the same row (and maybe throwing NOWAIT in there, then catching the errors and retrying), and things slow down.
On top of that, they have to actually update the row to mark it as locked, so the rest of your workers are sitting there waiting while one of them propagates its lock to disk (and the disks of however many servers you're replicating to). QueueClassic got some mileage out of the novel idea of randomly picking a row near the front of the queue to lock, but I can't still seem to get more than an an extra few hundred jobs per second out of it under heavy load.
So, many developers have started going straight t
