Skip to content

Instantly share code, notes, and snippets.

@ultimatecoder
Last active March 30, 2017 13:17
Show Gist options
  • Save ultimatecoder/f1ae6433b5cfe1337146716c0ddecc4b to your computer and use it in GitHub Desktop.
Save ultimatecoder/f1ae6433b5cfe1337146716c0ddecc4b to your computer and use it in GitHub Desktop.
This is the bot I have written for downloading and counting the data of National stock exchange. The bot is written when I was learning Python. The code is just to kept here for a reference and does not provide any garentee.
#! /usr/bin/python
"""
A bot National-Stock-Exchange-Robot. This is the robot for downloading and
counting the total number of lines of the csv file of the particular date. This
robot will remove days on which National Stock Exchange do not have the report.
User can also provide the list of dates separated by comma as
command line arguments.
$ ./BhavBot.py 2014-10-17 --holidays holiday.txt
or
$ ./BhavBot.py 2014-10-17
Dependency
==========
mechanize: https://pypi.python.org/pypi/mechanize/0.2.5
"""
import mechanize
import zipfile
from datetime import date, datetime
from sys import argv, exit
class Date():
def __init__(self, strdate, path=None):
self._date = datetime.strptime(strdate, '%Y-%m-%d').date()
self._mindate = date(day=21, month=11, year=1994)
if path:
self._holidays = self._getholidays(path)
else:
self._holidays = None
def _getholidays(self, path):
f = open(path, 'r')
list = f.read().split(',')
f.close()
# For removing file default '/n' escape sequence.
list[-1] = list[-1].strip()
return list
def validate(self):
weekday = self._date.strftime('%a')
strdate = self._date.strftime('%d-%b-%Y')
if (self._date > date.today()) or (self._date < self._mindate) or (weekday in ['Sun', 'Sat']) or (self._holidays and (strdate in self._holidays)):
return False
else:
return True
def getCombined(self):
return self._date.strftime('%d%b%Y')
def getSeperated(self):
return self._date.strftime('%d-%m-%Y')
class ZCFile():
def __init__(self, rawzip, strdate):
self._pathzip = '/tmp/data.csv.zip'
self._createzip(rawzip)
self._date = strdate
def _createzip(self, rawzip):
f = open(self._pathzip, 'w')
f.write(rawzip)
f.close()
def _readcsvFromZip(self):
self._zfile = zipfile.ZipFile(self._pathzip, 'r')
csv = self._zfile.read('cm%sbhav.csv' % (self._date), 'r')
self._zfile.close()
return csv
def count(self):
csv = self._readcsvFromZip()
return csv.count('\n')
class BhavBot():
def __init__(self, combinedate, seperatedate):
self._combinedate = combinedate.upper()
self._seperatedate = seperatedate
self._br = mechanize.Browser()
self._config()
self._hostname = 'http://nseindia.com'
self._url = self._hostname + '/ArchieveSearch?h_filetype=eqbhav&date=%s&section=EQ' % (self._seperatedate)
self._csv_url = self._hostname + '/content/historical/EQUITIES/%s/%s/cm%sbhav.csv.zip' % (self._seperatedate[-4:],self._combinedate[2:5], self._combinedate)
def _config(self):
user_agent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/37.0.2062.120 Chrome/37.0.2062.120 Safari/537.36"
accept = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
self._br.set_handle_robots(False)
self._br.addheaders = [('User-agent', user_agent), ('Accept', accept)]
def _getPage(self):
response = self._br.open(self._url)
return response.read()
def _getzip(self):
response = self._br.open(self._csv_url)
return response.read()
def _getTotalLines(self):
rawzip = self._getzip()
csv = ZCFile(rawzip, self._combinedate)
return csv.count()
def total_lines(self):
page = self._getPage()
if 'No file found' in page:
return False
else:
return self._getTotalLines()
def main():
usage = 'usage: date(YYYY-MM-DD) [--holidays File]'
args = argv[1:]
if not args:
print(usage)
exit(1)
else:
if len(args) == 3:
if args[1] == '--holidays':
mydate = Date(args[0], args[2])
else:
print(usage)
exit(1)
else:
mydate = Date(args[0])
if not mydate.validate():
print('ERROR: Stock exchange is not Open on this date. Ex: Before the current date or date which is not on Sunday or Saturday. Minimum date is 21-11-1994')
exit(1)
else:
bot = BhavBot(mydate.getCombined(), mydate.getSeperated())
print("Note: Please Wait. Downloading a file.")
lines = bot.total_lines()
if lines:
print('This CSV file contains : {}'.format(lines))
exit(0)
else:
print('ERROR: The csv is not available for the perticular date')
exit(1)
if __name__ == '__main__':
main()
01-Jan-2010,26-Jan-2010,12-Feb-2010,01-Mar-2010,16-Mar-2010,24-Mar-2010,01-Apr-2010,02-Apr-2010,14-Apr-2010,27-May-2010,01-Jul-2010,10-Sep-2010,30-Sep-2010,05-Nov-2010,17-Nov-2010,17-Dec-2010,26-Jan-2011,16-Feb-2011,02-Mar-2011,01-Apr-2011,04-Apr-2011,12-Apr-2011,14-Apr-2011,22-Apr-2011,17-May-2011,01-Jul-2011,15-Aug-2011,19-Aug-2011,31-Aug-2011,01-Sep-2011,30-Sep-2011,06-Oct-2011,26-Oct-2011,27-Oct-2011,07-Nov-2011,10-Nov-2011,06-Dec-2011,26-Jan-2012,16-Feb-2012,20-Feb-2012,08-Mar-2012,23-Mar-2012,02-Apr-2012,05-Apr-2012,06-Apr-2012,01-May-2012,02-Jul-2012,15-Aug-2012,20-Aug-2012,19-Sep-2012,02-Oct-2012,24-Oct-2012,13-Nov-2012,14-Nov-2012,28-Nov-2012,25-Dec-2012,25-Jan-2013,19-Feb-2013,27-Mar-2013,29-Mar-2013,01-Apr-2013,11-Apr-2013,19-Apr-2013,24-Apr-2013,01-May-2013,01-Jul-2013,09-Aug-2013,15-Aug-2013,09-Sep-2013,02-Oct-2013,16-Oct-2013,04-Nov-2013,14-Nov-2013,15-Nov-2013,25-Dec-2013,14-Jan-2014,19-Feb-2014,27-Feb-2014,17-Mar-2014,31-Mar-2014,01-Apr-2014,08-Apr-2014,14-Apr-2014,18-Apr-2014,24-Apr-2014,01-May-2014,14-May-2014,01-Jul-2014,29-Jul-2014,15-Aug-2014,18-Aug-2014,29-Aug-2014,02-Oct-2014,03-Oct-2014,06-Oct-2014,15-Oct-2014
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment