01_bc_day_04.md

For Python Quants Bootcamp

This is the Gist the For Python Quants Bootcamp in London 24. November 2017 (http://fpq.io)

This fourth boocamp day is about Python for Algorithmic Trading (II).

Topics

• prediction with deep learning
• streaming data & visualization
• Oanda trading platform
• deployment and automation

Beware the Robots

https://www.youtube.com/watch?v=fRj34o4hN4I

The Python Quants

http://twitter.com/dyjh

team@tpq.io | http://fpq.io

http://pyalgo.tpq.io | http://certificate.tpq.io

02_neural_network_python.ipynb

03_deep_learning.ipynb

04_eikon_cufflinks.ipynb

05_oanda_basics.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src='http://hilpisch.com/tpq_logo.png' width=\"300px\" align=\"right\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# FPQ Bootcamp &mdash; Day 4"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Algo Trading with Oanda**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from pylab import plt\n",
    "plt.style.use('seaborn')\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## configparser"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import configparser"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "c = configparser.ConfigParser()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "c.read('../fpq.cfg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "c['oanda_v20']['account_id']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## The tpqoa Wrapper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tpqoa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# oanda = tpqoa.tpqoa('../fpq.cfg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "oanda = tpqoa.tpqoa('../pyalgo.cfg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# oanda.get_instruments?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "oanda.get_instruments()[:7]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "raw = oanda.get_history('DE30_EUR', '2017-1-1', '2017-3-31', 'D', 'A')\n",
    "raw.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "raw = oanda.get_history('DE30_EUR', '2017-11-21', '2017-11-22', 'M1', 'A')\n",
    "raw.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "raw.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "raw['c'].plot(figsize=(10, 6));"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# oanda.stream_data('DE30_EUR')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class OandaCollector(tpqoa.tpqoa):\n",
    "    def __init__(self, *args):\n",
    "        super(OandaCollector, self).__init__(*args)\n",
    "        self.raw_data = pd.DataFrame()\n",
    "        self.position = 0\n",
    "        self.units = 10\n",
    "    def stream_data(self, instrument, stop=None):\n",
    "        self.stream_instrument = instrument\n",
    "        self.ticks = 0\n",
    "        response = self.ctx_stream.pricing.stream(\n",
    "            self.account_id, snapshot=True,\n",
    "            instruments=instrument)\n",
    "        for msg_type, msg in response.parts():\n",
    "            # print(msg_type, msg)\n",
    "            if msg_type == 'pricing.Price':\n",
    "                self.ticks +=1\n",
    "                try:\n",
    "                    self.on_success(msg.time,\n",
    "                                    float(msg.bids[0].price),\n",
    "                                    float(msg.asks[0].price))\n",
    "                except:\n",
    "                    pass\n",
    "                if stop is not None:\n",
    "                    if self.ticks >= stop:\n",
    "                        print('CLOSING OUT.')\n",
    "                        if self.position == 1:\n",
    "                            self.create_order(self.stream_instrument, -self.units)\n",
    "                            break\n",
    "                        elif self.position == -1:\n",
    "                            self.create_order(self.stream_instrument, self.units)\n",
    "                            break\n",
    "                        else:\n",
    "                            break\n",
    "    def on_success(self, time, bid, ask):\n",
    "        print(self.ticks, end=',')\n",
    "        # print('received:', time, bid, ask)\n",
    "        time = pd.Timestamp(time)\n",
    "        self.raw_data = self.raw_data.append(\n",
    "                    pd.DataFrame({'bid': bid, 'ask': ask}, index=[time]))\n",
    "        self.resam_data = self.raw_data.resample('5s', label='right').last().ffill()\n",
    "        self.resam_data['mid'] = self.resam_data.mean(axis=1)\n",
    "        self.resam_data['SMA1'] = self.resam_data['mid'].rolling(5).mean()\n",
    "        self.resam_data['SMA2'] = self.resam_data['mid'].rolling(10).mean()\n",
    "        if len(self.resam_data) > 6:\n",
    "            if (self.resam_data['SMA1'].iloc[-2] > self.resam_data['SMA2'].iloc[-2]\n",
    "                and self.position in [0, -1]):\n",
    "                if self.position == 0:\n",
    "                    units = self.units\n",
    "                elif self.position == -1:\n",
    "                    units = 2 * self.units\n",
    "                print('\\n\\ngoing long the market')\n",
    "                self.create_order(self.stream_instrument, units)\n",
    "                self.position = 1\n",
    "            elif (self.resam_data['SMA1'].iloc[-2] < self.resam_data['SMA2'].iloc[-2]\n",
    "                  and self.position in [0, 1]):\n",
    "                if self.position == 0:\n",
    "                    units = -self.units\n",
    "                elif self.position == 1:\n",
    "                    units = -2 * self.units\n",
    "                print('\\n\\ngoing short the market')\n",
    "                self.create_order(self.stream_instrument, units)\n",
    "                self.position = -1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "oc = OandaCollector('../pyalgo.cfg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "oc.stream_data('DE30_EUR', stop=350)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "oc.resam_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "oc.raw_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "oc.resam_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Placing Buy and Sell Orders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "oanda.create_order('DE30_EUR', 50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "oanda.create_order('DE30_EUR', -1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src='http://hilpisch.com/tpq_logo.png' width=\"300px\" align=\"right\">"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}

06_oanda_streaming_plot.ipynb

plyst.py

#
# plyst -- A Simple Plotly Streaming Wrapper Class
#
# For illustration purposes only
#
# The codes contained herein come without warranties or representations,
# to the extent permitted by applicable law.
#
# Read the RISK DISCLAIMER carefully.
#
# (c) The Python Quants GmbH
#
import configparser
import plotly.plotly as ply
from plotly.graph_objs import *

c = configparser.ConfigParser()
c.read('../pyalgo.cfg')
stream_ids = c['plotly']['api_tokens'].split(',')


class plyst(object):
    def __init__(self, streams=1, title=''):
        self.streams = {}
        self.scatters = []
        self.sockets = {}
        for stream in range(streams):
            self.streams[stream] = Stream(maxpoints=100,
                                          token=stream_ids[stream])
            self.scatters.append(Scatter(x=[], y=[],
                                         name='stream_%d' % stream,
                                         mode='lines+markers',
                                         stream=self.streams[stream]))
            self.sockets[stream] = ply.Stream(stream_ids[stream])
            self.sockets[stream].open()
        self.data = Data(self.scatters)
        self.layout = Layout(title=title)
        self.figure = Figure(data=self.data, layout=self.layout)
        self.fig_object = ply.plot(self.figure,
                                   filename='stream_plot_%s' % title,
                                   auto_open=False)

    def get_figure(self):
        return self.fig_object

tick_client.py

#
# Simple Tick Data Client with ZeroMQ
#
import zmq
import datetime

context = zmq.Context()
socket = context.socket(zmq.SUB)
socket.connect('tcp://127.0.0.1:5555')

socket.setsockopt_string(zmq.SUBSCRIBE, 'AAPL')

while True:
    msg = socket.recv_string()
    t = datetime.datetime.now()
    print('%s | ' % str(t), msg)

tick_collector.py

#
# Simple Tick Data Collector with ZeroMQ & pandas
#
import zmq
import datetime
import pandas as pd

context = zmq.Context()
socket = context.socket(zmq.SUB)
socket.connect('tcp://127.0.0.1:5555')

socket.setsockopt_string(zmq.SUBSCRIBE, 'AAPL')

raw = pd.DataFrame()

i = 0
while True:
    msg = socket.recv_string()
    i += 1
    t = datetime.datetime.now()
    print('%s | ' % str(t), msg)
    sym, value = msg.split()
    raw = raw.append(pd.DataFrame({sym: float(value)}, index=[t]))
    resam = raw.resample('5s', label='right').last()
    if i % 10 == 0:
        print('\n', resam.tail(3), '\n')

tick_server.py

#
# Simple Tick Data Server with ZeroMQ
#
import zmq
import time
import random

context = zmq.Context()
socket = context.socket(zmq.PUB)
socket.bind('tcp://127.0.0.1:5555')

AAPL = 100.

while True:
    AAPL += random.gauss(0, 1) * 0.5
    msg = 'AAPL %.4f' % AAPL
    socket.send_string(msg)
    print(msg)
    time.sleep(random.random() * 2)

tick_sma_trader.py

#
# Simple Tick Data Collector with ZeroMQ & pandas
#
import zmq
import datetime
import pandas as pd

context = zmq.Context()
socket = context.socket(zmq.SUB)
socket.connect('tcp://127.0.0.1:5555')

socket.setsockopt_string(zmq.SUBSCRIBE, 'AAPL')

raw = pd.DataFrame()

i = 0
position = 0
while True:
    msg = socket.recv_string()
    i += 1
    t = datetime.datetime.now()
    print('%s | ' % str(t), msg)
    sym, value = msg.split()
    raw = raw.append(pd.DataFrame({sym: float(value)}, index=[t]))
    resam = raw.resample('5s', label='right').last()
    resam['SMA1'] = resam[sym].rolling(3).mean()
    resam['SMA2'] = resam[sym].rolling(6).mean()
    if i % 10 == 0:
        print('\n', resam.tail(3), '\n')
    if len(resam) > 6:
        if resam['SMA1'].iloc[-2] > resam['SMA2'].iloc[-2] and position in [0, -1]:
            print('\ngoing long the market\n')
            position = 1
            # place trading logic here
        if resam['SMA1'].iloc[-2] < resam['SMA2'].iloc[-2] and position in [0, 1]:
            print('\ngoing short the market\n')
            position = -1
            # place trading logic here
 

tpqoa.py

#
# tpqoa is a wrapper class for the
# Oanda v20 API (RESTful & streaming)
# (c) Dr. Yves J. Hilpisch
# The Python Quants GmbH
#
# dependencies:
# pip install v20
# conda install pyyaml
#
import v20
import pandas as pd
import datetime as dt
import configparser

class tpqoa(object):
    ''' tpqoa is a Python wrapper class for the Oanda v20 API. '''

    def __init__(self, conf_file):
        ''' Init function expecting a configuration file with
        the following content:

        [oanda_v20]
        account_id = XYZ-ABC-...
        access_token = ZYXCAB...

        Parameters
        ==========
        conf_file: string
            path to and filename of the configuration file, e.g. '/home/me/oanda.cfg'
        '''
        self.config = configparser.ConfigParser()
        self.config.read(conf_file)
        self.access_token = self.config['oanda_v20']['access_token']
        self.account_id = self.config['oanda_v20']['account_id']
        self.ctx = v20.Context(
            hostname='api-fxpractice.oanda.com',
            port=443,
            ssl=True,
            application='sample_code',
            token=self.access_token,
            datetime_format='RFC3339')
        self.ctx_stream = v20.Context(
            hostname='stream-fxpractice.oanda.com',
            port=443,
            ssl=True,
            application='sample_code',
            token=self.access_token,
            datetime_format='RFC3339'
        )
        self.suffix = '.000000000Z'

    def get_instruments(self):
        ''' Retrieves and returns all instruments for the given account. '''
        resp = self.ctx.account.instruments(self.account_id)
        instruments = resp.get('instruments')
        instruments = [ins.dict() for ins in instruments]
        instruments = [(ins['displayName'], ins['name'])
                      for ins in instruments]
        return instruments

    def transform_datetime(self, dt):
        ''' Transforms Python datetime object to string. '''
        if isinstance(dt, str):
            dt = pd.Timestamp(dt).to_pydatetime()
        return dt.isoformat('T') + self.suffix
    
    def retrieve_data(self, instrument, start, end, granularity, price):
        raw = self.ctx.instrument.candles(
            instrument=instrument,
            fromTime=start, toTime=end,
            granularity=granularity, price=price)
        raw = raw.get('candles')
        raw = [cs.dict() for cs in raw]
        for cs in raw:
            cs.update(cs['ask'])
            del cs['ask']
        if len(raw) == 0:
            return pd.DataFrame() # return empty DataFrame is no data
        data = pd.DataFrame(raw)
        data['time'] = pd.to_datetime(data['time'])
        data = data.set_index('time')
        data.index = pd.DatetimeIndex(data.index)
        for col in list('ohlc'):
            data[col] = data[col].astype(float)
        return data

    def get_history(self, instrument, start, end,
                    granularity, price):
        ''' Retrieves historical data for instrument.

        Parameters
        ==========
        instrument: string
            valid instrument name
        start, end: datetime, str
            Python datetime or string objects for start and end
        granularity: string
            a string like 'S5', 'M1' or 'D'
        price: string
            one of 'A' (ask) or 'B' (bid)

        Returns
        =======
        data: pd.DataFrame
            pandas DataFrame object with data
        '''
        if granularity.startswith('S'):
            data = pd.DataFrame()
            dr = pd.date_range(start, end, freq='4h')
            for t in range(len(dr) - 1):
                start = self.transform_datetime(dr[t])
                end = self.transform_datetime(dr[t + 1])
                batch = self.retrieve_data(instrument, start, end,
                                          granularity, price)
                data = data.append(batch)
        else:
            start = self.transform_datetime(start)
            end = self.transform_datetime(end)
            data = self.retrieve_data(instrument, start, end,
                                      granularity, price)
        
        return data[['o', 'h', 'l', 'c', 'complete', 'volume']]

    def create_order(self, instrument, units):
        ''' Places order with Oanda.

        Parameters
        ==========
        instrument: string
            valid instrument name
        units: int
            number of units of instrument to be bought (positive int, eg 'units=50')
            or to be sold (negative int, eg 'units=-100')
        '''
        request = self.ctx.order.market(
            self.account_id,
            instrument=instrument,
            units=units,
        )
        order = request.get('orderFillTransaction')
        print('\n\n', order.dict(), '\n')

    def stream_data(self, instrument, stop=None):
        ''' Starts a real-time data stream.

        Parameters
        ==========
        instrument: string
            valid instrument name
        '''
        self.stream_instrument = instrument
        self.ticks = 0
        response = self.ctx_stream.pricing.stream(
            self.account_id, snapshot=True,
            instruments=instrument)
        for msg_type, msg in response.parts():
            # print(msg_type, msg)
            if msg_type == 'pricing.Price':
                self.ticks +=1
                self.on_success(msg.time,
                                float(msg.bids[0].price),
                                float(msg.asks[0].price))
                if stop is not None:
                    if self.ticks >= stop:
                        break

    def on_success(self, time, bid, ask):
        ''' Method called when new data is retrieved. '''
        print(time, bid, ask)

    def get_account_summary(self, detailed=False):
        ''' Returns summary data for Oanda account.'''
        if detailed is True:
            response = self.ctx.account.get(self.account_id)
        else:
            response = self.ctx.account.summary(self.account_id)
        raw = response.get('account')
        return raw.dict()

    def get_transactions(self, tid=0):
        ''' Retrieves and returns transactions data. '''
        response = self.ctx.transaction.since(self.account_id, id=tid)
        transactions = response.get('transactions')
        transactions = [t.dict() for t in transactions]
        return transactions

    def print_transactions(self, tid=0):
        ''' Prints basic transactions data. '''
        transactions = self.get_transactions(tid)
        for trans in transactions:
             templ = '%5s | %s | %9s | %12s'
             print(templ % (trans['id'],
                            trans['time'],
                            trans['instrument'],
                            trans['units']))