Skip to content

Instantly share code, notes, and snippets.

@zaydek-old
Last active July 2, 2019 05:02
Show Gist options
  • Save zaydek-old/2b8747459d87e04d82d33d1335d41d13 to your computer and use it in GitHub Desktop.
Save zaydek-old/2b8747459d87e04d82d33d1335d41d13 to your computer and use it in GitHub Desktop.
Example of some source code: this quotes the stock market (e.g. <2k stocks) at one-second intervals for all unique quotes throughout the day
$ head AAPL
09:30:00.000 166.6400
09:30:02.000 166.3000
09:30:03.000 166.2400
09:30:04.000 166.2050
09:30:05.000 166.2900
09:30:06.000 166.1500
09:30:07.000 166.2450
09:30:08.000 166.0675
09:30:09.000 166.0200
$ head TSLA
09:30:00.000 348.4375
09:30:03.000 348.4000
09:30:04.000 348.1650
09:30:05.000 348.4400
09:30:07.000 347.9000
09:30:14.000 347.9300
09:30:15.000 348.1575
09:30:19.000 347.8900
09:30:20.000 348.3750
$ ls
A AMD BABA BW CLVS CYS EA EWZ FTK HALO IGT KBH MA MWA OBE PGX RL SKX SYF TSN VEU WUBA
AA AME BAC BWA CMA CYTX EAT EXAS FTNT HAS IJH KBR MAC MXIM OC PHM RLGY SLB SYK TSRO VFC WY
AABA AMGN BAH BX CMC CZR EBAY EXC FTR HBAN IJR KEM MAR MYGN OCLR PIR RLJ SLCA SYMBL TSS VG WYN
AAL AMH BAX BYD CMCM CZZ EC EXEL FTV HBI ILF KERX MARA MYL OCN PK ROKU SLM SYMC TSU VGK WYNN
AAOI AMJ BB BZUN CMCSA D ECA EXK FWONK HCA ILG KEY MAS MYSZ OCSL PLAY ROST SLV SYN TTI VIAB X
...
package main
import (
"fmt"
"io"
"log"
"net/http"
"os"
"sync"
"time"
"owl.delivery/code/decode"
"owl.delivery/code/mkt"
"owl.delivery/code/rbh"
"owl.delivery/code/sh"
"owl.delivery/code/tick"
"owl.delivery/code/ws"
)
var (
// these vars control the speed and window for which the program runs, e.g.
//
// 09:29:55
// 09:29:56
// ...
// 15:59:58
// 15:59:59
d = 1 * time.Second // iterate one second at a time
t1 = mkt.Now().Add(1 * time.Second) // mkt.Open.Add(-5 * time.Second) // start at market open -5 seconds
t2 = t1.Add(5 * time.Second) // mkt.Close // stop at market close
)
// init sets up program's usage, logging, and http timeout.
func init() {
if len(os.Args) == 2 && os.Args[1] == "--help" { // in case the program is invoked as $ program --help
fmt.Fprintf(os.Stderr, "e.g. %s FB AAPL AMZN NFLX GOOG\n", os.Args[0])
os.Exit(2)
}
log.SetFlags(log.Lshortfile) // custom logging; e.g. main.go:y:x: ...
http.DefaultClient.Timeout = d // set a global http timeout to d duration
}
// send is used for sending mulitple values over a channel.
type send struct {
t time.Time // e.g. 2006-01-02 09:30:00 -0500 EST
s string // e.g. "09:30:00" *optimization* so each for-loop does not need to precompute the same string
q rbh.Quotes
}
// This program is designed to quote up to <2k stocks at 1-second intervals from market open to market close.
// It is comprised of 6 core steps:
//
// 1. Prepare program vars.
// 2. Create and open a directory for which to cache quotes and errors.
// 3. Create a file for all symbols and for-loop errors. For for-loop errors, also write to stderr.
// 4. Establish a TCP connection so the first quote is not lost.
// 5. Create a channel to streamline output and separate concerns.
// - the goroutine below the channel is the producer.
// - step 6. is the channel's consumer.
// 6. Create a filter that filters non-unique quotes. For each unique quote, cache to its corresponding file.
func main() {
// 1.
var (
SYMBLs = os.Args[1:] // symbols from the command-line, e.g. {"FB", "AAPL", "AMZN", "NFLX", "GOOG"}
quotesURL = rbh.QuotesURL(SYMBLs...) // pre-generate url, e.g. "https://...?symbols=FB,AAPL,AMZN,NFLX,GOOG"
quotes rbh.Quotes
)
// 2.
date := mkt.Now().Format("2006-01-02") // today's date
for _, fn := range []func(string) error{sh.Mkdir, sh.Chdir} { // e.g. $ mkdir ... && cd ...
if err := fn(date); err != nil {
log.Fatalln(err)
}
}
// 3.
outs := make(map[string]io.WriteCloser) // map of writers
var err error
for _, SYMBL := range append(SYMBLs, "errs") { // append "errs" to cache for-loop errors
if outs[SYMBL], err = os.Create(SYMBL); err != nil { // every writer has a corresponding file to write to
log.Fatalln(err)
}
defer outs[SYMBL].Close()
}
errs := io.MultiWriter(os.Stderr, outs["errs"]) // also write for-loop errs to stderr
// 4.
for { // we cannot continue until we establish a tcp connection
if _, err = ws.Get(rbh.URL); err == nil {
break // tcp cached; break for-loop
}
fmt.Fprintln(errs, mkt.Now().Format("15:04:05.000"), err)
}
// 5.
outputC := make(chan send)
go func() {
defer close(outputC)
for t := range tick.New(d, t1, t2) {
// 5a.
if err := decode.Get(quotesURL, &quotes); err != nil { // reuse tcp conn
fmt.Fprintln(errs, t.Format("15:04:05.000"), err)
continue
}
// 5b.
t = t.In(mkt.Loc).Truncate(d) // needed? program speed used to be designed for milliseconds
outputC <- send{t, t.Format("15:04:05"), quotes} // precompute time-string
}
}()
// 6.
nuanced := nuance() // generate a filter that determines if a quote is unique or not; omit non-unique quotes
for recv := range outputC {
for _, quote := range recv.q.Quotes {
if !recv.t.Before(mkt.Open) && nuanced(quote) {
fmt.Fprintf(outs[quote.SYMBL], "%s,%s\n", recv.s, quote.Quote) // only write unique quotes after t1 time
}
}
}
}
// nuance returns a closure that takes a quote and returns whether the quote is unique.
// nuance stores one quote at a time.
func nuance() func(rbh.Quote) bool {
// 1.
var mu sync.Mutex // needed? program used to be concurrent
var mp = make(map[string]string)
// 2.
return func(q rbh.Quote) bool {
// 2a.
mu.Lock()
defer mu.Unlock()
// 2b.
if mp[q.SYMBL] != q.Quote {
mp[q.SYMBL] = q.Quote
return true
} else {
return false
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment