Skip to content

Instantly share code, notes, and snippets.

@meta-ks
Created July 22, 2023 08:19
Show Gist options
  • Save meta-ks/6356e373711283174c106213909ae490 to your computer and use it in GitHub Desktop.
Save meta-ks/6356e373711283174c106213909ae490 to your computer and use it in GitHub Desktop.
NSE utilities. Download data from NSE usign jtrader
import os
import time
import io
import requests
nse_urls = {
'N50':'',
'N100':'',
'N200':'',
'N500':'https://archives.nseindia.com/content/indices/ind_nifty500list.csv',
'TOP1000':'https://static.nseindia.com//s3fs-public/inline-files/MCAP_31032020_TOP1000.xlsx', #older data
'N_ALL':'https://static.nseindia.com//s3fs-public/inline-files/MCAP31122022_0.xlsx', #recent
}
def download_nse_data(nse_url):
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:108.0) Gecko/20100101 Firefox/108.0',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.5',
'DNT': '1',
'Connection': 'keep-alive',
'Upgrade-Insecure-Requests': '1',
'Sec-Fetch-Dest': 'document',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-Site': 'none',
'Sec-Fetch-User': '?1',
'Pragma': 'no-cache',
'Cache-Control': 'no-cache',
}
res = requests.get(nse_url, headers=headers)
if(res.status_code == 200):
print(f'[*]Got <200> at: {nse_url}')
return res
def get_BSE_all_active_equities():
bse_all_scrips_url = 'https://api.bseindia.com/BseIndiaAPI/api/LitsOfScripCSVDownload/w'
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:108.0) Gecko/20100101 Firefox/108.0',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.5',
# 'Accept-Encoding': 'gzip, deflate, br',
'Connection': 'keep-alive',
'Referer': 'https://www.bseindia.com/',
'Upgrade-Insecure-Requests': '1',
'Sec-Fetch-Dest': 'document',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-Site': 'same-site',
'Sec-Fetch-User': '?1',
'Pragma': 'no-cache',
'Cache-Control': 'no-cache',
}
params = {
'segment': 'Equity',
'status': 'Active',
'industry': '',
'Group': '',
'Scripcode': '',
}
csv_res = requests.get(bse_all_scrips_url, params=params, headers=headers)
if(csv_res.status_code == 200):
print(f'[*]Got <200> at: {bse_all_scrips_url}')
df = pd.read_csv(io.StringIO(csv_res.text))
else:
print(f'[-]Got {csv_res.status_code} at: {bse_all_scrips_url}.....')
df = None
return df
def nse_scrip_info(scrip):
#Need to debug this routine; nse response is unpredictable,
#Better use jugaad-data or nsepy and get the quote page
print(f'--------------------I AM NOT WORKING PROPERLY--------------')
nse_equity_url = 'https://www.nseindia.com/api/quote-equity'
#Somehow using referer as variable is not wroking, omitting for now
# referer_url = f'https://www.nseindia.com/get-quotes/equity?symbol={scrip}'
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:108.0) Gecko/20100101 Firefox/108.0',
'Accept': '*/*',
'Accept-Language': 'en-US,en;q=0.5',
'Connection': 'keep-alive',
# 'Referer': referer_url,
'Sec-Fetch-Dest': 'empty',
'Sec-Fetch-Mode': 'cors',
'Sec-Fetch-Site': 'same-origin',
'Pragma': 'no-cache',
'Cache-Control': 'no-cache',
}
params = {'symbol': scrip}
try:
#Cookies are omitted as of now. Can add later if req
res = requests.get(nse_equity_url, params=params, headers=headers)
json_d = res.json()
except Exception as e:
print(f'[-]Failed to fetch {scrip}. Got <{res.status_code}>: {e}: {res.content}')
return None
return json_d
def get_N500():
csv_res = download_nse_data(nse_urls['N500'])
df = pd.read_csv(io.StringIO(csv_res.text))
# df = pd.read_csv(nse_urls['N500'])
# print('\nN500 data:\n--------------------------\n')
# print(df)
return df
def get_Ntop1000():
xlx_res = download_nse_data(nse_urls['TOP1000'])
with io.BytesIO(xlx_res.content) as fh:
df = pd.io.excel.read_excel(fh)
# df = pd.read_excel(nse_urls['TOP1000'])
# print('\nN_TOP_1000 data:\n--------------------------\n')
# print(df)
return df
def get_nifty_all():
xlx_res = download_nse_data(nse_urls['N_ALL'])
with io.BytesIO(xlx_res.content) as fh:
df = pd.io.excel.read_excel(fh)
# df = pd.read_excel(nse_urls['N_ALL'])
# print('\nN_All data:\n--------------------------\n')
# print(df)
return df
def download_equity_data_from_nse(nifty_equities=None, top_n=1000, refresh_data=False, pickle_df=False):
from jugaad_data.nse import NSELive
from utils.serializer_utils import unpickle_py_obj, pickle_py_obj
if nifty_equities is not None:
n_all = nifty_equities
else:
n_all = get_nifty_all()
pkl_dest_path = 'NSE_data/'
if not os.path.exists(pkl_dest_path):
print(f'[*]Creating destination dir: {pkl_dest_path}')
os.makedirs(pkl_dest_path)
pkl_files = os.listdir(pkl_dest_path)
top_scrips = list(n_all['Symbol'][:top_n])
n = NSELive()
rows_list = []
for ix,scrip in enumerate(top_scrips):
fp = f'NSE_data/{scrip}'
empty_pkl = False
try:
if not refresh_data:
# print(f'[{ix}]Unpickling: {scrip}')
q = unpickle_py_obj(fp)
if q.get('msg'):
print(f'[-]Missing data: {scrip}: {q}')
# empty_pkl = True
except Exception as e:
print(f'[-]Error: {e}')
empty_pkl = True
if empty_pkl or refresh_data or scrip not in pkl_files:
q = n.stock_quote(scrip)
pickle_py_obj(q, fp, print_stat=False)
print(f'[{ix+1}/{top_n}]Pickled: {scrip}')
time.sleep(1)
#The list is in order of MCap
mcap_rank = ix+1
nifty_slice = f'Nifty {50*(ix//50+1)}'
try:
mcap_in_1000cr = round(n_all.query(f"Symbol=='{scrip}'")['Market capitalization as on December 30, 2022\n(Rs in Lakhs)'].item()/100000, 3)
except Exception as e:
mcap_in_1000cr = 0
print(f'[-]Error in MCap: {e}')
info = q.get('info', {})
sym = scrip
# sym = info.get('symbol')
meta = q.get('metadata', {})
lst_date = meta.get('listingDate')
sector_index = meta.get('pdSectorInd')
sec_info = q.get('securityInfo', {})
is_derivatives = sec_info.get('derivatives')
p_info = q.get('priceInfo', {})
pband = p_info.get('pPriceBand')
if pband:
pband = int(pband.replace('No Band', '0'))
lowerCP = p_info.get('lowerCP')
upperCP = p_info.get('upperCP')
ind_info = q.get('industryInfo', {})
macro = ind_info.get('macro')
sector = ind_info.get('sector')
industry = ind_info.get('industry')
basic_ind = ind_info.get('basicIndustry')
eq_dict = {
'Symbol':sym, 'Nifty Slice': f'{mcap_rank}: {nifty_slice}',
# 'rank':mcap_rank, 'nifty slice':nifty_slice,
'Macro':macro, 'Sector':sector, 'Industry':industry, 'Basic Industry':basic_ind,
'Sector Index': sector_index, 'Market Cap (in 1000Cr)':mcap_in_1000cr,
'Price Band':pband, 'MCap Rank': mcap_rank, 'Nifty x':nifty_slice
}
rows_list.append(eq_dict)
eq_df = pd.DataFrame(rows_list)
if pickle_df:
pickle_py_obj(eq_df, EQUITIES_NSE_DATA_FILEPATH)
# pd.set_option('display.max_rows', 20)
return eq_df
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment