bse-scrap.go

package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/gocarina/gocsv"
)

type Data struct {
	Deal         string `csv:"Deal Date" json:"deal_date"`
	SecurityCode string `csv:"Security Code" json:"security_code"`
	Company      string `csv:"Company" json:"company"`
	ClientName   string `csv:"Client Name" json:"client_name"`
	DealType     string `csv:"Deal Type" json:"deal_type"`
	Quantity     string `csv:"Quantity" json:"quantity"`
	Price        string `csv:"Price" json:"price"`
}

func main() {

	date := time.Now().Format("01Jan2006")

	d := flag.String("date", date, "Enter date for which you want the data")

	flag.Parse()

	u := fmt.Sprintf("http://www.bseindia.com/markets/downloads/Bulk_%s.csv", *d)

	fmt.Println("URL: ", u)
	req, err := http.NewRequest("GET", u, nil)
	if err != nil {
		log.Fatalln("error occurred in creating request: %v", err)
	}
	req.Header.Set("Host", "www.bseindia.com")
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	req.Header.Set("Referer", "http://www.bseindia.com/markets/equity/EQReports/BulknBlockDeals.aspx")
	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		log.Fatalln("error in sending request: %v", err)
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		log.Fatalln("status code is not 200")
	}

	f, _ := ioutil.ReadAll(resp.Body)

	anomalyRemoval := time.Now()
	fstr := strings.Replace(string(f), ",,", ",", -1)
	fstr = strings.Replace(fstr, ",LTD", "LTD", -1)
	fstr = strings.Replace(fstr, ", LTD", "LTD", -1)

	fmt.Printf("Anomaly removal took: %s\n", time.Since(anomalyRemoval))

	clients := []*Data{}
	csvunmarshalTime := time.Now()
	if err := gocsv.UnmarshalString(fstr, &clients); err != nil { // Load clients from file
		panic(err)
	}

	fmt.Printf("CSV Unmarshal took %s\n", time.Since(csvunmarshalTime))

	out, err := os.Create("bse_bulk.json")
	if err != nil {
		log.Fatalln("error in writing json: %v", err)
	}
	defer out.Close()

	jsonEncodingTime := time.Now()
	err = json.NewEncoder(out).Encode(clients)
	if err != nil {
		log.Fatalln("error in writing json: %v", err)
	}

	fmt.Printf("JSON encoding took: %s\n", time.Since(jsonEncodingTime))

}

nse-scrap.go

package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/PuerkitoBio/goquery"

	"github.com/gocarina/gocsv"
)

type Data struct {
	Date               string `csv:"Date" json:"date"`
	Symbol             string `csv:"Symbol" json:"symbol"`
	SecurityName       string `csv:"Security Name" json:"security_name"`
	ClientName         string `csv:"Client Name" json:"client_name"`
	BuySell            string `csv:"Buy / Sell" json:"buy_sell"`
	QuantityTraded     string `csv:"Quantity Traded " json:"quantity_traded"`
	TradePriceAvgPrice string `csv:"Trade Price / Wght. Avg. Price" json:"trade_price_wght_avg_price"`
	Remarks            string `csv:"Remarks" json:"remarks"`
}

func main() {

	req, err := http.NewRequest("GET", "https://www.nseindia.com/products/dynaContent/equities/equities/bulkdeals.jsp?symbol=&segmentLink=13&symbolCount=&dateRange=day&fromDate=&toDate=&dataType=DEALS", nil)
	if err != nil {
		log.Fatalln("error occurred in creating request: %v", err)
	}
	req.Header.Set("Host", "www.nseindia.com")
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		log.Fatalln("error in sending request: %v", err)
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		log.Fatalln("status code is not 200")
	}

	doc, err := goquery.NewDocumentFromResponse(resp)
	if err != nil {
		log.Fatalln("error in creating document: %v", err)
	}

	csvData := doc.Find("#csvContentDiv").Text()

	csvData = strings.Replace(csvData, ":", "\n", -1)

	anomalyRemoval := time.Now()
	fstr := strings.Replace(csvData, ",,", ",", -1)
	fstr = strings.Replace(fstr, ",LTD", "LTD", -1)
	fstr = strings.Replace(fstr, ", LTD", "LTD", -1)

	fmt.Printf("Anomaly removal took: %s\n", time.Since(anomalyRemoval))

	clients := []*Data{}
	csvunmarshalTime := time.Now()
	if err := gocsv.UnmarshalString(fstr, &clients); err != nil { // Load clients from file
		panic(err)
	}

	fmt.Printf("CSV Unmarshal took %s\n", time.Since(csvunmarshalTime))

	out, err := os.Create("nse_bulk.json")
	if err != nil {
		log.Fatalln("error in writing json: %v", err)
	}
	defer out.Close()

	jsonEncodingTime := time.Now()
	err = json.NewEncoder(out).Encode(clients)
	if err != nil {
		log.Fatalln("error in writing json: %v", err)
	}

	fmt.Printf("JSON encoding took: %s\n", time.Since(jsonEncodingTime))

}