Last active
March 30, 2017 13:17
-
-
Save ultimatecoder/f1ae6433b5cfe1337146716c0ddecc4b to your computer and use it in GitHub Desktop.
This is the bot I have written for downloading and counting the data of National stock exchange. The bot is written when I was learning Python. The code is just to kept here for a reference and does not provide any garentee.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#! /usr/bin/python | |
""" | |
A bot National-Stock-Exchange-Robot. This is the robot for downloading and | |
counting the total number of lines of the csv file of the particular date. This | |
robot will remove days on which National Stock Exchange do not have the report. | |
User can also provide the list of dates separated by comma as | |
command line arguments. | |
$ ./BhavBot.py 2014-10-17 --holidays holiday.txt | |
or | |
$ ./BhavBot.py 2014-10-17 | |
Dependency | |
========== | |
mechanize: https://pypi.python.org/pypi/mechanize/0.2.5 | |
""" | |
import mechanize | |
import zipfile | |
from datetime import date, datetime | |
from sys import argv, exit | |
class Date(): | |
def __init__(self, strdate, path=None): | |
self._date = datetime.strptime(strdate, '%Y-%m-%d').date() | |
self._mindate = date(day=21, month=11, year=1994) | |
if path: | |
self._holidays = self._getholidays(path) | |
else: | |
self._holidays = None | |
def _getholidays(self, path): | |
f = open(path, 'r') | |
list = f.read().split(',') | |
f.close() | |
# For removing file default '/n' escape sequence. | |
list[-1] = list[-1].strip() | |
return list | |
def validate(self): | |
weekday = self._date.strftime('%a') | |
strdate = self._date.strftime('%d-%b-%Y') | |
if (self._date > date.today()) or (self._date < self._mindate) or (weekday in ['Sun', 'Sat']) or (self._holidays and (strdate in self._holidays)): | |
return False | |
else: | |
return True | |
def getCombined(self): | |
return self._date.strftime('%d%b%Y') | |
def getSeperated(self): | |
return self._date.strftime('%d-%m-%Y') | |
class ZCFile(): | |
def __init__(self, rawzip, strdate): | |
self._pathzip = '/tmp/data.csv.zip' | |
self._createzip(rawzip) | |
self._date = strdate | |
def _createzip(self, rawzip): | |
f = open(self._pathzip, 'w') | |
f.write(rawzip) | |
f.close() | |
def _readcsvFromZip(self): | |
self._zfile = zipfile.ZipFile(self._pathzip, 'r') | |
csv = self._zfile.read('cm%sbhav.csv' % (self._date), 'r') | |
self._zfile.close() | |
return csv | |
def count(self): | |
csv = self._readcsvFromZip() | |
return csv.count('\n') | |
class BhavBot(): | |
def __init__(self, combinedate, seperatedate): | |
self._combinedate = combinedate.upper() | |
self._seperatedate = seperatedate | |
self._br = mechanize.Browser() | |
self._config() | |
self._hostname = 'http://nseindia.com' | |
self._url = self._hostname + '/ArchieveSearch?h_filetype=eqbhav&date=%s§ion=EQ' % (self._seperatedate) | |
self._csv_url = self._hostname + '/content/historical/EQUITIES/%s/%s/cm%sbhav.csv.zip' % (self._seperatedate[-4:],self._combinedate[2:5], self._combinedate) | |
def _config(self): | |
user_agent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/37.0.2062.120 Chrome/37.0.2062.120 Safari/537.36" | |
accept = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8' | |
self._br.set_handle_robots(False) | |
self._br.addheaders = [('User-agent', user_agent), ('Accept', accept)] | |
def _getPage(self): | |
response = self._br.open(self._url) | |
return response.read() | |
def _getzip(self): | |
response = self._br.open(self._csv_url) | |
return response.read() | |
def _getTotalLines(self): | |
rawzip = self._getzip() | |
csv = ZCFile(rawzip, self._combinedate) | |
return csv.count() | |
def total_lines(self): | |
page = self._getPage() | |
if 'No file found' in page: | |
return False | |
else: | |
return self._getTotalLines() | |
def main(): | |
usage = 'usage: date(YYYY-MM-DD) [--holidays File]' | |
args = argv[1:] | |
if not args: | |
print(usage) | |
exit(1) | |
else: | |
if len(args) == 3: | |
if args[1] == '--holidays': | |
mydate = Date(args[0], args[2]) | |
else: | |
print(usage) | |
exit(1) | |
else: | |
mydate = Date(args[0]) | |
if not mydate.validate(): | |
print('ERROR: Stock exchange is not Open on this date. Ex: Before the current date or date which is not on Sunday or Saturday. Minimum date is 21-11-1994') | |
exit(1) | |
else: | |
bot = BhavBot(mydate.getCombined(), mydate.getSeperated()) | |
print("Note: Please Wait. Downloading a file.") | |
lines = bot.total_lines() | |
if lines: | |
print('This CSV file contains : {}'.format(lines)) | |
exit(0) | |
else: | |
print('ERROR: The csv is not available for the perticular date') | |
exit(1) | |
if __name__ == '__main__': | |
main() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
01-Jan-2010,26-Jan-2010,12-Feb-2010,01-Mar-2010,16-Mar-2010,24-Mar-2010,01-Apr-2010,02-Apr-2010,14-Apr-2010,27-May-2010,01-Jul-2010,10-Sep-2010,30-Sep-2010,05-Nov-2010,17-Nov-2010,17-Dec-2010,26-Jan-2011,16-Feb-2011,02-Mar-2011,01-Apr-2011,04-Apr-2011,12-Apr-2011,14-Apr-2011,22-Apr-2011,17-May-2011,01-Jul-2011,15-Aug-2011,19-Aug-2011,31-Aug-2011,01-Sep-2011,30-Sep-2011,06-Oct-2011,26-Oct-2011,27-Oct-2011,07-Nov-2011,10-Nov-2011,06-Dec-2011,26-Jan-2012,16-Feb-2012,20-Feb-2012,08-Mar-2012,23-Mar-2012,02-Apr-2012,05-Apr-2012,06-Apr-2012,01-May-2012,02-Jul-2012,15-Aug-2012,20-Aug-2012,19-Sep-2012,02-Oct-2012,24-Oct-2012,13-Nov-2012,14-Nov-2012,28-Nov-2012,25-Dec-2012,25-Jan-2013,19-Feb-2013,27-Mar-2013,29-Mar-2013,01-Apr-2013,11-Apr-2013,19-Apr-2013,24-Apr-2013,01-May-2013,01-Jul-2013,09-Aug-2013,15-Aug-2013,09-Sep-2013,02-Oct-2013,16-Oct-2013,04-Nov-2013,14-Nov-2013,15-Nov-2013,25-Dec-2013,14-Jan-2014,19-Feb-2014,27-Feb-2014,17-Mar-2014,31-Mar-2014,01-Apr-2014,08-Apr-2014,14-Apr-2014,18-Apr-2014,24-Apr-2014,01-May-2014,14-May-2014,01-Jul-2014,29-Jul-2014,15-Aug-2014,18-Aug-2014,29-Aug-2014,02-Oct-2014,03-Oct-2014,06-Oct-2014,15-Oct-2014 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment