Skip to content

Instantly share code, notes, and snippets.

@ishanjain28
Last active Sep 25, 2017
Embed
What would you like to do?
package main
import (
"encoding/json"
"flag"
"fmt"
"io/ioutil"
"log"
"net/http"
"os"
"strings"
"time"
"github.com/gocarina/gocsv"
)
type Data struct {
Deal string `csv:"Deal Date" json:"deal_date"`
SecurityCode string `csv:"Security Code" json:"security_code"`
Company string `csv:"Company" json:"company"`
ClientName string `csv:"Client Name" json:"client_name"`
DealType string `csv:"Deal Type" json:"deal_type"`
Quantity string `csv:"Quantity" json:"quantity"`
Price string `csv:"Price" json:"price"`
}
func main() {
date := time.Now().Format("01Jan2006")
d := flag.String("date", date, "Enter date for which you want the data")
flag.Parse()
u := fmt.Sprintf("http://www.bseindia.com/markets/downloads/Bulk_%s.csv", *d)
fmt.Println("URL: ", u)
req, err := http.NewRequest("GET", u, nil)
if err != nil {
log.Fatalln("error occurred in creating request: %v", err)
}
req.Header.Set("Host", "www.bseindia.com")
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
req.Header.Set("Referer", "http://www.bseindia.com/markets/equity/EQReports/BulknBlockDeals.aspx")
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
log.Fatalln("error in sending request: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
log.Fatalln("status code is not 200")
}
f, _ := ioutil.ReadAll(resp.Body)
anomalyRemoval := time.Now()
fstr := strings.Replace(string(f), ",,", ",", -1)
fstr = strings.Replace(fstr, ",LTD", "LTD", -1)
fstr = strings.Replace(fstr, ", LTD", "LTD", -1)
fmt.Printf("Anomaly removal took: %s\n", time.Since(anomalyRemoval))
clients := []*Data{}
csvunmarshalTime := time.Now()
if err := gocsv.UnmarshalString(fstr, &clients); err != nil { // Load clients from file
panic(err)
}
fmt.Printf("CSV Unmarshal took %s\n", time.Since(csvunmarshalTime))
out, err := os.Create("bse_bulk.json")
if err != nil {
log.Fatalln("error in writing json: %v", err)
}
defer out.Close()
jsonEncodingTime := time.Now()
err = json.NewEncoder(out).Encode(clients)
if err != nil {
log.Fatalln("error in writing json: %v", err)
}
fmt.Printf("JSON encoding took: %s\n", time.Since(jsonEncodingTime))
}
package main
import (
"encoding/json"
"fmt"
"log"
"net/http"
"os"
"strings"
"time"
"github.com/PuerkitoBio/goquery"
"github.com/gocarina/gocsv"
)
type Data struct {
Date string `csv:"Date" json:"date"`
Symbol string `csv:"Symbol" json:"symbol"`
SecurityName string `csv:"Security Name" json:"security_name"`
ClientName string `csv:"Client Name" json:"client_name"`
BuySell string `csv:"Buy / Sell" json:"buy_sell"`
QuantityTraded string `csv:"Quantity Traded " json:"quantity_traded"`
TradePriceAvgPrice string `csv:"Trade Price / Wght. Avg. Price" json:"trade_price_wght_avg_price"`
Remarks string `csv:"Remarks" json:"remarks"`
}
func main() {
req, err := http.NewRequest("GET", "https://www.nseindia.com/products/dynaContent/equities/equities/bulkdeals.jsp?symbol=&segmentLink=13&symbolCount=&dateRange=day&fromDate=&toDate=&dataType=DEALS", nil)
if err != nil {
log.Fatalln("error occurred in creating request: %v", err)
}
req.Header.Set("Host", "www.nseindia.com")
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
log.Fatalln("error in sending request: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
log.Fatalln("status code is not 200")
}
doc, err := goquery.NewDocumentFromResponse(resp)
if err != nil {
log.Fatalln("error in creating document: %v", err)
}
csvData := doc.Find("#csvContentDiv").Text()
csvData = strings.Replace(csvData, ":", "\n", -1)
anomalyRemoval := time.Now()
fstr := strings.Replace(csvData, ",,", ",", -1)
fstr = strings.Replace(fstr, ",LTD", "LTD", -1)
fstr = strings.Replace(fstr, ", LTD", "LTD", -1)
fmt.Printf("Anomaly removal took: %s\n", time.Since(anomalyRemoval))
clients := []*Data{}
csvunmarshalTime := time.Now()
if err := gocsv.UnmarshalString(fstr, &clients); err != nil { // Load clients from file
panic(err)
}
fmt.Printf("CSV Unmarshal took %s\n", time.Since(csvunmarshalTime))
out, err := os.Create("nse_bulk.json")
if err != nil {
log.Fatalln("error in writing json: %v", err)
}
defer out.Close()
jsonEncodingTime := time.Now()
err = json.NewEncoder(out).Encode(clients)
if err != nil {
log.Fatalln("error in writing json: %v", err)
}
fmt.Printf("JSON encoding took: %s\n", time.Since(jsonEncodingTime))
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment